xref: /netbsd-src/sys/kern/vfs_bio.c (revision e39ef1d61eee3ccba837ee281f1e098c864487aa)
1 /*	$NetBSD: vfs_bio.c,v 1.232 2011/10/05 01:53:03 jakllsch Exp $	*/
2 
3 /*-
4  * Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Andrew Doran, and by Wasabi Systems, Inc.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 /*-
33  * Copyright (c) 1982, 1986, 1989, 1993
34  *	The Regents of the University of California.  All rights reserved.
35  * (c) UNIX System Laboratories, Inc.
36  * All or some portions of this file are derived from material licensed
37  * to the University of California by American Telephone and Telegraph
38  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
39  * the permission of UNIX System Laboratories, Inc.
40  *
41  * Redistribution and use in source and binary forms, with or without
42  * modification, are permitted provided that the following conditions
43  * are met:
44  * 1. Redistributions of source code must retain the above copyright
45  *    notice, this list of conditions and the following disclaimer.
46  * 2. Redistributions in binary form must reproduce the above copyright
47  *    notice, this list of conditions and the following disclaimer in the
48  *    documentation and/or other materials provided with the distribution.
49  * 3. Neither the name of the University nor the names of its contributors
50  *    may be used to endorse or promote products derived from this software
51  *    without specific prior written permission.
52  *
53  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
54  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
55  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
56  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
57  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
58  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
59  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
60  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
61  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
62  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
63  * SUCH DAMAGE.
64  *
65  *	@(#)vfs_bio.c	8.6 (Berkeley) 1/11/94
66  */
67 
68 /*-
69  * Copyright (c) 1994 Christopher G. Demetriou
70  *
71  * Redistribution and use in source and binary forms, with or without
72  * modification, are permitted provided that the following conditions
73  * are met:
74  * 1. Redistributions of source code must retain the above copyright
75  *    notice, this list of conditions and the following disclaimer.
76  * 2. Redistributions in binary form must reproduce the above copyright
77  *    notice, this list of conditions and the following disclaimer in the
78  *    documentation and/or other materials provided with the distribution.
79  * 3. All advertising materials mentioning features or use of this software
80  *    must display the following acknowledgement:
81  *	This product includes software developed by the University of
82  *	California, Berkeley and its contributors.
83  * 4. Neither the name of the University nor the names of its contributors
84  *    may be used to endorse or promote products derived from this software
85  *    without specific prior written permission.
86  *
87  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
88  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
89  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
90  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
91  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
92  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
93  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
94  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
95  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
96  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
97  * SUCH DAMAGE.
98  *
99  *	@(#)vfs_bio.c	8.6 (Berkeley) 1/11/94
100  */
101 
102 /*
103  * The buffer cache subsystem.
104  *
105  * Some references:
106  *	Bach: The Design of the UNIX Operating System (Prentice Hall, 1986)
107  *	Leffler, et al.: The Design and Implementation of the 4.3BSD
108  *		UNIX Operating System (Addison Welley, 1989)
109  *
110  * Locking
111  *
112  * There are three locks:
113  * - bufcache_lock: protects global buffer cache state.
114  * - BC_BUSY: a long term per-buffer lock.
115  * - buf_t::b_objlock: lock on completion (biowait vs biodone).
116  *
117  * For buffers associated with vnodes (a most common case) b_objlock points
118  * to the vnode_t::v_interlock.  Otherwise, it points to generic buffer_lock.
119  *
120  * Lock order:
121  *	bufcache_lock ->
122  *		buf_t::b_objlock
123  */
124 
125 #include <sys/cdefs.h>
126 __KERNEL_RCSID(0, "$NetBSD: vfs_bio.c,v 1.232 2011/10/05 01:53:03 jakllsch Exp $");
127 
128 #include "opt_bufcache.h"
129 
130 #include <sys/param.h>
131 #include <sys/systm.h>
132 #include <sys/kernel.h>
133 #include <sys/proc.h>
134 #include <sys/buf.h>
135 #include <sys/vnode.h>
136 #include <sys/mount.h>
137 #include <sys/resourcevar.h>
138 #include <sys/sysctl.h>
139 #include <sys/conf.h>
140 #include <sys/kauth.h>
141 #include <sys/fstrans.h>
142 #include <sys/intr.h>
143 #include <sys/cpu.h>
144 #include <sys/wapbl.h>
145 #include <sys/bitops.h>
146 
147 #include <uvm/uvm.h>	/* extern struct uvm uvm */
148 
149 #include <miscfs/specfs/specdev.h>
150 
151 #ifndef	BUFPAGES
152 # define BUFPAGES 0
153 #endif
154 
155 #ifdef BUFCACHE
156 # if (BUFCACHE < 5) || (BUFCACHE > 95)
157 #  error BUFCACHE is not between 5 and 95
158 # endif
159 #else
160 # define BUFCACHE 15
161 #endif
162 
163 u_int	nbuf;			/* desired number of buffer headers */
164 u_int	bufpages = BUFPAGES;	/* optional hardwired count */
165 u_int	bufcache = BUFCACHE;	/* max % of RAM to use for buffer cache */
166 
167 /* Function prototypes */
168 struct bqueue;
169 
170 static void buf_setwm(void);
171 static int buf_trim(void);
172 static void *bufpool_page_alloc(struct pool *, int);
173 static void bufpool_page_free(struct pool *, void *);
174 static buf_t *bio_doread(struct vnode *, daddr_t, int,
175     kauth_cred_t, int);
176 static buf_t *getnewbuf(int, int, int);
177 static int buf_lotsfree(void);
178 static int buf_canrelease(void);
179 static u_long buf_mempoolidx(u_long);
180 static u_long buf_roundsize(u_long);
181 static void *buf_alloc(size_t);
182 static void buf_mrelease(void *, size_t);
183 static void binsheadfree(buf_t *, struct bqueue *);
184 static void binstailfree(buf_t *, struct bqueue *);
185 #ifdef DEBUG
186 static int checkfreelist(buf_t *, struct bqueue *, int);
187 #endif
188 static void biointr(void *);
189 static void biodone2(buf_t *);
190 static void bref(buf_t *);
191 static void brele(buf_t *);
192 static void sysctl_kern_buf_setup(void);
193 static void sysctl_vm_buf_setup(void);
194 
195 /*
196  * Definitions for the buffer hash lists.
197  */
198 #define	BUFHASH(dvp, lbn)	\
199 	(&bufhashtbl[(((long)(dvp) >> 8) + (int)(lbn)) & bufhash])
200 LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash;
201 u_long	bufhash;
202 struct bqueue bufqueues[BQUEUES];
203 
204 static kcondvar_t needbuffer_cv;
205 
206 /*
207  * Buffer queue lock.
208  */
209 kmutex_t bufcache_lock;
210 kmutex_t buffer_lock;
211 
212 /* Software ISR for completed transfers. */
213 static void *biodone_sih;
214 
215 /* Buffer pool for I/O buffers. */
216 static pool_cache_t buf_cache;
217 static pool_cache_t bufio_cache;
218 
219 #define MEMPOOL_INDEX_OFFSET (ilog2(DEV_BSIZE))	/* smallest pool is 512 bytes */
220 #define NMEMPOOLS (ilog2(MAXBSIZE) - MEMPOOL_INDEX_OFFSET + 1)
221 __CTASSERT((1 << (NMEMPOOLS + MEMPOOL_INDEX_OFFSET - 1)) == MAXBSIZE);
222 
223 /* Buffer memory pools */
224 static struct pool bmempools[NMEMPOOLS];
225 
226 static struct vm_map *buf_map;
227 
228 /*
229  * Buffer memory pool allocator.
230  */
231 static void *
232 bufpool_page_alloc(struct pool *pp, int flags)
233 {
234 
235 	return (void *)uvm_km_alloc(buf_map,
236 	    MAXBSIZE, MAXBSIZE,
237 	    ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT | UVM_KMF_TRYLOCK)
238 	    | UVM_KMF_WIRED);
239 }
240 
241 static void
242 bufpool_page_free(struct pool *pp, void *v)
243 {
244 
245 	uvm_km_free(buf_map, (vaddr_t)v, MAXBSIZE, UVM_KMF_WIRED);
246 }
247 
248 static struct pool_allocator bufmempool_allocator = {
249 	.pa_alloc = bufpool_page_alloc,
250 	.pa_free = bufpool_page_free,
251 	.pa_pagesz = MAXBSIZE,
252 };
253 
254 /* Buffer memory management variables */
255 u_long bufmem_valimit;
256 u_long bufmem_hiwater;
257 u_long bufmem_lowater;
258 u_long bufmem;
259 
260 /*
261  * MD code can call this to set a hard limit on the amount
262  * of virtual memory used by the buffer cache.
263  */
264 int
265 buf_setvalimit(vsize_t sz)
266 {
267 
268 	/* We need to accommodate at least NMEMPOOLS of MAXBSIZE each */
269 	if (sz < NMEMPOOLS * MAXBSIZE)
270 		return EINVAL;
271 
272 	bufmem_valimit = sz;
273 	return 0;
274 }
275 
276 static void
277 buf_setwm(void)
278 {
279 
280 	bufmem_hiwater = buf_memcalc();
281 	/* lowater is approx. 2% of memory (with bufcache = 15) */
282 #define	BUFMEM_WMSHIFT	3
283 #define	BUFMEM_HIWMMIN	(64 * 1024 << BUFMEM_WMSHIFT)
284 	if (bufmem_hiwater < BUFMEM_HIWMMIN)
285 		/* Ensure a reasonable minimum value */
286 		bufmem_hiwater = BUFMEM_HIWMMIN;
287 	bufmem_lowater = bufmem_hiwater >> BUFMEM_WMSHIFT;
288 }
289 
290 #ifdef DEBUG
291 int debug_verify_freelist = 0;
292 static int
293 checkfreelist(buf_t *bp, struct bqueue *dp, int ison)
294 {
295 	buf_t *b;
296 
297 	if (!debug_verify_freelist)
298 		return 1;
299 
300 	TAILQ_FOREACH(b, &dp->bq_queue, b_freelist) {
301 		if (b == bp)
302 			return ison ? 1 : 0;
303 	}
304 
305 	return ison ? 0 : 1;
306 }
307 #endif
308 
309 /*
310  * Insq/Remq for the buffer hash lists.
311  * Call with buffer queue locked.
312  */
313 static void
314 binsheadfree(buf_t *bp, struct bqueue *dp)
315 {
316 
317 	KASSERT(mutex_owned(&bufcache_lock));
318 	KASSERT(bp->b_freelistindex == -1);
319 	TAILQ_INSERT_HEAD(&dp->bq_queue, bp, b_freelist);
320 	dp->bq_bytes += bp->b_bufsize;
321 	bp->b_freelistindex = dp - bufqueues;
322 }
323 
324 static void
325 binstailfree(buf_t *bp, struct bqueue *dp)
326 {
327 
328 	KASSERT(mutex_owned(&bufcache_lock));
329 	KASSERT(bp->b_freelistindex == -1);
330 	TAILQ_INSERT_TAIL(&dp->bq_queue, bp, b_freelist);
331 	dp->bq_bytes += bp->b_bufsize;
332 	bp->b_freelistindex = dp - bufqueues;
333 }
334 
335 void
336 bremfree(buf_t *bp)
337 {
338 	struct bqueue *dp;
339 	int bqidx = bp->b_freelistindex;
340 
341 	KASSERT(mutex_owned(&bufcache_lock));
342 
343 	KASSERT(bqidx != -1);
344 	dp = &bufqueues[bqidx];
345 	KDASSERT(checkfreelist(bp, dp, 1));
346 	KASSERT(dp->bq_bytes >= bp->b_bufsize);
347 	TAILQ_REMOVE(&dp->bq_queue, bp, b_freelist);
348 	dp->bq_bytes -= bp->b_bufsize;
349 
350 	/* For the sysctl helper. */
351 	if (bp == dp->bq_marker)
352 		dp->bq_marker = NULL;
353 
354 #if defined(DIAGNOSTIC)
355 	bp->b_freelistindex = -1;
356 #endif /* defined(DIAGNOSTIC) */
357 }
358 
359 /*
360  * Add a reference to an buffer structure that came from buf_cache.
361  */
362 static inline void
363 bref(buf_t *bp)
364 {
365 
366 	KASSERT(mutex_owned(&bufcache_lock));
367 	KASSERT(bp->b_refcnt > 0);
368 
369 	bp->b_refcnt++;
370 }
371 
372 /*
373  * Free an unused buffer structure that came from buf_cache.
374  */
375 static inline void
376 brele(buf_t *bp)
377 {
378 
379 	KASSERT(mutex_owned(&bufcache_lock));
380 	KASSERT(bp->b_refcnt > 0);
381 
382 	if (bp->b_refcnt-- == 1) {
383 		buf_destroy(bp);
384 #ifdef DEBUG
385 		memset((char *)bp, 0, sizeof(*bp));
386 #endif
387 		pool_cache_put(buf_cache, bp);
388 	}
389 }
390 
391 /*
392  * note that for some ports this is used by pmap bootstrap code to
393  * determine kva size.
394  */
395 u_long
396 buf_memcalc(void)
397 {
398 	u_long n;
399 
400 	/*
401 	 * Determine the upper bound of memory to use for buffers.
402 	 *
403 	 *	- If bufpages is specified, use that as the number
404 	 *	  pages.
405 	 *
406 	 *	- Otherwise, use bufcache as the percentage of
407 	 *	  physical memory.
408 	 */
409 	if (bufpages != 0) {
410 		n = bufpages;
411 	} else {
412 		if (bufcache < 5) {
413 			printf("forcing bufcache %d -> 5", bufcache);
414 			bufcache = 5;
415 		}
416 		if (bufcache > 95) {
417 			printf("forcing bufcache %d -> 95", bufcache);
418 			bufcache = 95;
419 		}
420 		n = calc_cache_size(buf_map, bufcache,
421 		    (buf_map != kernel_map) ? 100 : BUFCACHE_VA_MAXPCT)
422 		    / PAGE_SIZE;
423 	}
424 
425 	n <<= PAGE_SHIFT;
426 	if (bufmem_valimit != 0 && n > bufmem_valimit)
427 		n = bufmem_valimit;
428 
429 	return (n);
430 }
431 
432 /*
433  * Initialize buffers and hash links for buffers.
434  */
435 void
436 bufinit(void)
437 {
438 	struct bqueue *dp;
439 	int use_std;
440 	u_int i;
441 
442 	mutex_init(&bufcache_lock, MUTEX_DEFAULT, IPL_NONE);
443 	mutex_init(&buffer_lock, MUTEX_DEFAULT, IPL_NONE);
444 	cv_init(&needbuffer_cv, "needbuf");
445 
446 	if (bufmem_valimit != 0) {
447 		vaddr_t minaddr = 0, maxaddr;
448 		buf_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
449 					  bufmem_valimit, 0, false, 0);
450 		if (buf_map == NULL)
451 			panic("bufinit: cannot allocate submap");
452 	} else
453 		buf_map = kernel_map;
454 
455 	/*
456 	 * Initialize buffer cache memory parameters.
457 	 */
458 	bufmem = 0;
459 	buf_setwm();
460 
461 	/* On "small" machines use small pool page sizes where possible */
462 	use_std = (physmem < atop(16*1024*1024));
463 
464 	/*
465 	 * Also use them on systems that can map the pool pages using
466 	 * a direct-mapped segment.
467 	 */
468 #ifdef PMAP_MAP_POOLPAGE
469 	use_std = 1;
470 #endif
471 
472 	buf_cache = pool_cache_init(sizeof(buf_t), 0, 0, 0,
473 	    "bufpl", NULL, IPL_SOFTBIO, NULL, NULL, NULL);
474 	bufio_cache = pool_cache_init(sizeof(buf_t), 0, 0, 0,
475 	    "biopl", NULL, IPL_BIO, NULL, NULL, NULL);
476 
477 	bufmempool_allocator.pa_backingmap = buf_map;
478 	for (i = 0; i < NMEMPOOLS; i++) {
479 		struct pool_allocator *pa;
480 		struct pool *pp = &bmempools[i];
481 		u_int size = 1 << (i + MEMPOOL_INDEX_OFFSET);
482 		char *name = kmem_alloc(8, KM_SLEEP); /* XXX: never freed */
483 		if (__predict_false(size >= 1048576))
484 			(void)snprintf(name, 8, "buf%um", size / 1048576);
485 		else if (__predict_true(size >= 1024))
486 			(void)snprintf(name, 8, "buf%uk", size / 1024);
487 		else
488 			(void)snprintf(name, 8, "buf%ub", size);
489 		pa = (size <= PAGE_SIZE && use_std)
490 			? &pool_allocator_nointr
491 			: &bufmempool_allocator;
492 		pool_init(pp, size, 0, 0, 0, name, pa, IPL_NONE);
493 		pool_setlowat(pp, 1);
494 		pool_sethiwat(pp, 1);
495 	}
496 
497 	/* Initialize the buffer queues */
498 	for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++) {
499 		TAILQ_INIT(&dp->bq_queue);
500 		dp->bq_bytes = 0;
501 	}
502 
503 	/*
504 	 * Estimate hash table size based on the amount of memory we
505 	 * intend to use for the buffer cache. The average buffer
506 	 * size is dependent on our clients (i.e. filesystems).
507 	 *
508 	 * For now, use an empirical 3K per buffer.
509 	 */
510 	nbuf = (bufmem_hiwater / 1024) / 3;
511 	bufhashtbl = hashinit(nbuf, HASH_LIST, true, &bufhash);
512 
513 	sysctl_kern_buf_setup();
514 	sysctl_vm_buf_setup();
515 }
516 
517 void
518 bufinit2(void)
519 {
520 
521 	biodone_sih = softint_establish(SOFTINT_BIO | SOFTINT_MPSAFE, biointr,
522 	    NULL);
523 	if (biodone_sih == NULL)
524 		panic("bufinit2: can't establish soft interrupt");
525 }
526 
527 static int
528 buf_lotsfree(void)
529 {
530 	int try, thresh;
531 
532 	/* Always allocate if less than the low water mark. */
533 	if (bufmem < bufmem_lowater)
534 		return 1;
535 
536 	/* Never allocate if greater than the high water mark. */
537 	if (bufmem > bufmem_hiwater)
538 		return 0;
539 
540 	/* If there's anything on the AGE list, it should be eaten. */
541 	if (TAILQ_FIRST(&bufqueues[BQ_AGE].bq_queue) != NULL)
542 		return 0;
543 
544 	/*
545 	 * The probabily of getting a new allocation is inversely
546 	 * proportional to the current size of the cache, using
547 	 * a granularity of 16 steps.
548 	 */
549 	try = random() & 0x0000000fL;
550 
551 	/* Don't use "16 * bufmem" here to avoid a 32-bit overflow. */
552 	thresh = (bufmem - bufmem_lowater) /
553 	    ((bufmem_hiwater - bufmem_lowater) / 16);
554 
555 	if (try >= thresh)
556 		return 1;
557 
558 	/* Otherwise don't allocate. */
559 	return 0;
560 }
561 
562 /*
563  * Return estimate of bytes we think need to be
564  * released to help resolve low memory conditions.
565  *
566  * => called with bufcache_lock held.
567  */
568 static int
569 buf_canrelease(void)
570 {
571 	int pagedemand, ninvalid = 0;
572 
573 	KASSERT(mutex_owned(&bufcache_lock));
574 
575 	if (bufmem < bufmem_lowater)
576 		return 0;
577 
578 	if (bufmem > bufmem_hiwater)
579 		return bufmem - bufmem_hiwater;
580 
581 	ninvalid += bufqueues[BQ_AGE].bq_bytes;
582 
583 	pagedemand = uvmexp.freetarg - uvmexp.free;
584 	if (pagedemand < 0)
585 		return ninvalid;
586 	return MAX(ninvalid, MIN(2 * MAXBSIZE,
587 	    MIN((bufmem - bufmem_lowater) / 16, pagedemand * PAGE_SIZE)));
588 }
589 
590 /*
591  * Buffer memory allocation helper functions
592  */
593 static u_long
594 buf_mempoolidx(u_long size)
595 {
596 	u_int n = 0;
597 
598 	size -= 1;
599 	size >>= MEMPOOL_INDEX_OFFSET;
600 	while (size) {
601 		size >>= 1;
602 		n += 1;
603 	}
604 	if (n >= NMEMPOOLS)
605 		panic("buf mem pool index %d", n);
606 	return n;
607 }
608 
609 static u_long
610 buf_roundsize(u_long size)
611 {
612 	/* Round up to nearest power of 2 */
613 	return (1 << (buf_mempoolidx(size) + MEMPOOL_INDEX_OFFSET));
614 }
615 
616 static void *
617 buf_alloc(size_t size)
618 {
619 	u_int n = buf_mempoolidx(size);
620 	void *addr;
621 
622 	while (1) {
623 		addr = pool_get(&bmempools[n], PR_NOWAIT);
624 		if (addr != NULL)
625 			break;
626 
627 		/* No memory, see if we can free some. If so, try again */
628 		mutex_enter(&bufcache_lock);
629 		if (buf_drain(1) > 0) {
630 			mutex_exit(&bufcache_lock);
631 			continue;
632 		}
633 
634 		if (curlwp == uvm.pagedaemon_lwp) {
635 			mutex_exit(&bufcache_lock);
636 			return NULL;
637 		}
638 
639 		/* Wait for buffers to arrive on the LRU queue */
640 		cv_timedwait(&needbuffer_cv, &bufcache_lock, hz / 4);
641 		mutex_exit(&bufcache_lock);
642 	}
643 
644 	return addr;
645 }
646 
647 static void
648 buf_mrelease(void *addr, size_t size)
649 {
650 
651 	pool_put(&bmempools[buf_mempoolidx(size)], addr);
652 }
653 
654 /*
655  * bread()/breadn() helper.
656  */
657 static buf_t *
658 bio_doread(struct vnode *vp, daddr_t blkno, int size, kauth_cred_t cred,
659     int async)
660 {
661 	buf_t *bp;
662 	struct mount *mp;
663 
664 	bp = getblk(vp, blkno, size, 0, 0);
665 
666 #ifdef DIAGNOSTIC
667 	if (bp == NULL) {
668 		panic("bio_doread: no such buf");
669 	}
670 #endif
671 
672 	/*
673 	 * If buffer does not have data valid, start a read.
674 	 * Note that if buffer is BC_INVAL, getblk() won't return it.
675 	 * Therefore, it's valid if its I/O has completed or been delayed.
676 	 */
677 	if (!ISSET(bp->b_oflags, (BO_DONE | BO_DELWRI))) {
678 		/* Start I/O for the buffer. */
679 		SET(bp->b_flags, B_READ | async);
680 		if (async)
681 			BIO_SETPRIO(bp, BPRIO_TIMELIMITED);
682 		else
683 			BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
684 		VOP_STRATEGY(vp, bp);
685 
686 		/* Pay for the read. */
687 		curlwp->l_ru.ru_inblock++;
688 	} else if (async)
689 		brelse(bp, 0);
690 
691 	if (vp->v_type == VBLK)
692 		mp = vp->v_specmountpoint;
693 	else
694 		mp = vp->v_mount;
695 
696 	/*
697 	 * Collect statistics on synchronous and asynchronous reads.
698 	 * Reads from block devices are charged to their associated
699 	 * filesystem (if any).
700 	 */
701 	if (mp != NULL) {
702 		if (async == 0)
703 			mp->mnt_stat.f_syncreads++;
704 		else
705 			mp->mnt_stat.f_asyncreads++;
706 	}
707 
708 	return (bp);
709 }
710 
711 /*
712  * Read a disk block.
713  * This algorithm described in Bach (p.54).
714  */
715 int
716 bread(struct vnode *vp, daddr_t blkno, int size, kauth_cred_t cred,
717     int flags, buf_t **bpp)
718 {
719 	buf_t *bp;
720 	int error;
721 
722 	/* Get buffer for block. */
723 	bp = *bpp = bio_doread(vp, blkno, size, cred, 0);
724 
725 	/* Wait for the read to complete, and return result. */
726 	error = biowait(bp);
727 	if (error == 0 && (flags & B_MODIFY) != 0)
728 		error = fscow_run(bp, true);
729 
730 	return error;
731 }
732 
733 /*
734  * Read-ahead multiple disk blocks. The first is sync, the rest async.
735  * Trivial modification to the breada algorithm presented in Bach (p.55).
736  */
737 int
738 breadn(struct vnode *vp, daddr_t blkno, int size, daddr_t *rablks,
739     int *rasizes, int nrablks, kauth_cred_t cred, int flags, buf_t **bpp)
740 {
741 	buf_t *bp;
742 	int error, i;
743 
744 	bp = *bpp = bio_doread(vp, blkno, size, cred, 0);
745 
746 	/*
747 	 * For each of the read-ahead blocks, start a read, if necessary.
748 	 */
749 	mutex_enter(&bufcache_lock);
750 	for (i = 0; i < nrablks; i++) {
751 		/* If it's in the cache, just go on to next one. */
752 		if (incore(vp, rablks[i]))
753 			continue;
754 
755 		/* Get a buffer for the read-ahead block */
756 		mutex_exit(&bufcache_lock);
757 		(void) bio_doread(vp, rablks[i], rasizes[i], cred, B_ASYNC);
758 		mutex_enter(&bufcache_lock);
759 	}
760 	mutex_exit(&bufcache_lock);
761 
762 	/* Otherwise, we had to start a read for it; wait until it's valid. */
763 	error = biowait(bp);
764 	if (error == 0 && (flags & B_MODIFY) != 0)
765 		error = fscow_run(bp, true);
766 	return error;
767 }
768 
769 /*
770  * Block write.  Described in Bach (p.56)
771  */
772 int
773 bwrite(buf_t *bp)
774 {
775 	int rv, sync, wasdelayed;
776 	struct vnode *vp;
777 	struct mount *mp;
778 
779 	KASSERT(ISSET(bp->b_cflags, BC_BUSY));
780 	KASSERT(!cv_has_waiters(&bp->b_done));
781 
782 	vp = bp->b_vp;
783 	if (vp != NULL) {
784 		KASSERT(bp->b_objlock == vp->v_interlock);
785 		if (vp->v_type == VBLK)
786 			mp = vp->v_specmountpoint;
787 		else
788 			mp = vp->v_mount;
789 	} else {
790 		mp = NULL;
791 	}
792 
793 	if (mp && mp->mnt_wapbl) {
794 		if (bp->b_iodone != mp->mnt_wapbl_op->wo_wapbl_biodone) {
795 			bdwrite(bp);
796 			return 0;
797 		}
798 	}
799 
800 	/*
801 	 * Remember buffer type, to switch on it later.  If the write was
802 	 * synchronous, but the file system was mounted with MNT_ASYNC,
803 	 * convert it to a delayed write.
804 	 * XXX note that this relies on delayed tape writes being converted
805 	 * to async, not sync writes (which is safe, but ugly).
806 	 */
807 	sync = !ISSET(bp->b_flags, B_ASYNC);
808 	if (sync && mp != NULL && ISSET(mp->mnt_flag, MNT_ASYNC)) {
809 		bdwrite(bp);
810 		return (0);
811 	}
812 
813 	/*
814 	 * Collect statistics on synchronous and asynchronous writes.
815 	 * Writes to block devices are charged to their associated
816 	 * filesystem (if any).
817 	 */
818 	if (mp != NULL) {
819 		if (sync)
820 			mp->mnt_stat.f_syncwrites++;
821 		else
822 			mp->mnt_stat.f_asyncwrites++;
823 	}
824 
825 	/*
826 	 * Pay for the I/O operation and make sure the buf is on the correct
827 	 * vnode queue.
828 	 */
829 	bp->b_error = 0;
830 	wasdelayed = ISSET(bp->b_oflags, BO_DELWRI);
831 	CLR(bp->b_flags, B_READ);
832 	if (wasdelayed) {
833 		mutex_enter(&bufcache_lock);
834 		mutex_enter(bp->b_objlock);
835 		CLR(bp->b_oflags, BO_DONE | BO_DELWRI);
836 		reassignbuf(bp, bp->b_vp);
837 		mutex_exit(&bufcache_lock);
838 	} else {
839 		curlwp->l_ru.ru_oublock++;
840 		mutex_enter(bp->b_objlock);
841 		CLR(bp->b_oflags, BO_DONE | BO_DELWRI);
842 	}
843 	if (vp != NULL)
844 		vp->v_numoutput++;
845 	mutex_exit(bp->b_objlock);
846 
847 	/* Initiate disk write. */
848 	if (sync)
849 		BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
850 	else
851 		BIO_SETPRIO(bp, BPRIO_TIMELIMITED);
852 
853 	VOP_STRATEGY(vp, bp);
854 
855 	if (sync) {
856 		/* If I/O was synchronous, wait for it to complete. */
857 		rv = biowait(bp);
858 
859 		/* Release the buffer. */
860 		brelse(bp, 0);
861 
862 		return (rv);
863 	} else {
864 		return (0);
865 	}
866 }
867 
868 int
869 vn_bwrite(void *v)
870 {
871 	struct vop_bwrite_args *ap = v;
872 
873 	return (bwrite(ap->a_bp));
874 }
875 
876 /*
877  * Delayed write.
878  *
879  * The buffer is marked dirty, but is not queued for I/O.
880  * This routine should be used when the buffer is expected
881  * to be modified again soon, typically a small write that
882  * partially fills a buffer.
883  *
884  * NB: magnetic tapes cannot be delayed; they must be
885  * written in the order that the writes are requested.
886  *
887  * Described in Leffler, et al. (pp. 208-213).
888  */
889 void
890 bdwrite(buf_t *bp)
891 {
892 
893 	KASSERT(bp->b_vp == NULL || bp->b_vp->v_tag != VT_UFS ||
894 	    bp->b_vp->v_type == VBLK || ISSET(bp->b_flags, B_COWDONE));
895 	KASSERT(ISSET(bp->b_cflags, BC_BUSY));
896 	KASSERT(!cv_has_waiters(&bp->b_done));
897 
898 	/* If this is a tape block, write the block now. */
899 	if (bdev_type(bp->b_dev) == D_TAPE) {
900 		bawrite(bp);
901 		return;
902 	}
903 
904 	if (wapbl_vphaswapbl(bp->b_vp)) {
905 		struct mount *mp = wapbl_vptomp(bp->b_vp);
906 
907 		if (bp->b_iodone != mp->mnt_wapbl_op->wo_wapbl_biodone) {
908 			WAPBL_ADD_BUF(mp, bp);
909 		}
910 	}
911 
912 	/*
913 	 * If the block hasn't been seen before:
914 	 *	(1) Mark it as having been seen,
915 	 *	(2) Charge for the write,
916 	 *	(3) Make sure it's on its vnode's correct block list.
917 	 */
918 	KASSERT(bp->b_vp == NULL || bp->b_objlock == bp->b_vp->v_interlock);
919 
920 	if (!ISSET(bp->b_oflags, BO_DELWRI)) {
921 		mutex_enter(&bufcache_lock);
922 		mutex_enter(bp->b_objlock);
923 		SET(bp->b_oflags, BO_DELWRI);
924 		curlwp->l_ru.ru_oublock++;
925 		reassignbuf(bp, bp->b_vp);
926 		mutex_exit(&bufcache_lock);
927 	} else {
928 		mutex_enter(bp->b_objlock);
929 	}
930 	/* Otherwise, the "write" is done, so mark and release the buffer. */
931 	CLR(bp->b_oflags, BO_DONE);
932 	mutex_exit(bp->b_objlock);
933 
934 	brelse(bp, 0);
935 }
936 
937 /*
938  * Asynchronous block write; just an asynchronous bwrite().
939  */
940 void
941 bawrite(buf_t *bp)
942 {
943 
944 	KASSERT(ISSET(bp->b_cflags, BC_BUSY));
945 	KASSERT(bp->b_vp != NULL);
946 
947 	SET(bp->b_flags, B_ASYNC);
948 	VOP_BWRITE(bp->b_vp, bp);
949 }
950 
951 /*
952  * Release a buffer on to the free lists.
953  * Described in Bach (p. 46).
954  */
955 void
956 brelsel(buf_t *bp, int set)
957 {
958 	struct bqueue *bufq;
959 	struct vnode *vp;
960 
961 	KASSERT(mutex_owned(&bufcache_lock));
962 	KASSERT(!cv_has_waiters(&bp->b_done));
963 	KASSERT(bp->b_refcnt > 0);
964 
965 	SET(bp->b_cflags, set);
966 
967 	KASSERT(ISSET(bp->b_cflags, BC_BUSY));
968 	KASSERT(bp->b_iodone == NULL);
969 
970 	/* Wake up any processes waiting for any buffer to become free. */
971 	cv_signal(&needbuffer_cv);
972 
973 	/* Wake up any proceeses waiting for _this_ buffer to become */
974 	if (ISSET(bp->b_cflags, BC_WANTED))
975 		CLR(bp->b_cflags, BC_WANTED|BC_AGE);
976 
977 	/* If it's clean clear the copy-on-write flag. */
978 	if (ISSET(bp->b_flags, B_COWDONE)) {
979 		mutex_enter(bp->b_objlock);
980 		if (!ISSET(bp->b_oflags, BO_DELWRI))
981 			CLR(bp->b_flags, B_COWDONE);
982 		mutex_exit(bp->b_objlock);
983 	}
984 
985 	/*
986 	 * Determine which queue the buffer should be on, then put it there.
987 	 */
988 
989 	/* If it's locked, don't report an error; try again later. */
990 	if (ISSET(bp->b_flags, B_LOCKED))
991 		bp->b_error = 0;
992 
993 	/* If it's not cacheable, or an error, mark it invalid. */
994 	if (ISSET(bp->b_cflags, BC_NOCACHE) || bp->b_error != 0)
995 		SET(bp->b_cflags, BC_INVAL);
996 
997 	if (ISSET(bp->b_cflags, BC_VFLUSH)) {
998 		/*
999 		 * This is a delayed write buffer that was just flushed to
1000 		 * disk.  It is still on the LRU queue.  If it's become
1001 		 * invalid, then we need to move it to a different queue;
1002 		 * otherwise leave it in its current position.
1003 		 */
1004 		CLR(bp->b_cflags, BC_VFLUSH);
1005 		if (!ISSET(bp->b_cflags, BC_INVAL|BC_AGE) &&
1006 		    !ISSET(bp->b_flags, B_LOCKED) && bp->b_error == 0) {
1007 			KDASSERT(checkfreelist(bp, &bufqueues[BQ_LRU], 1));
1008 			goto already_queued;
1009 		} else {
1010 			bremfree(bp);
1011 		}
1012 	}
1013 
1014 	KDASSERT(checkfreelist(bp, &bufqueues[BQ_AGE], 0));
1015 	KDASSERT(checkfreelist(bp, &bufqueues[BQ_LRU], 0));
1016 	KDASSERT(checkfreelist(bp, &bufqueues[BQ_LOCKED], 0));
1017 
1018 	if ((bp->b_bufsize <= 0) || ISSET(bp->b_cflags, BC_INVAL)) {
1019 		/*
1020 		 * If it's invalid or empty, dissociate it from its vnode
1021 		 * and put on the head of the appropriate queue.
1022 		 */
1023 		if (ISSET(bp->b_flags, B_LOCKED)) {
1024 			if (wapbl_vphaswapbl(vp = bp->b_vp)) {
1025 				struct mount *mp = wapbl_vptomp(vp);
1026 
1027 				KASSERT(bp->b_iodone
1028 				    != mp->mnt_wapbl_op->wo_wapbl_biodone);
1029 				WAPBL_REMOVE_BUF(mp, bp);
1030 			}
1031 		}
1032 
1033 		mutex_enter(bp->b_objlock);
1034 		CLR(bp->b_oflags, BO_DONE|BO_DELWRI);
1035 		if ((vp = bp->b_vp) != NULL) {
1036 			KASSERT(bp->b_objlock == vp->v_interlock);
1037 			reassignbuf(bp, bp->b_vp);
1038 			brelvp(bp);
1039 			mutex_exit(vp->v_interlock);
1040 		} else {
1041 			KASSERT(bp->b_objlock == &buffer_lock);
1042 			mutex_exit(bp->b_objlock);
1043 		}
1044 
1045 		if (bp->b_bufsize <= 0)
1046 			/* no data */
1047 			goto already_queued;
1048 		else
1049 			/* invalid data */
1050 			bufq = &bufqueues[BQ_AGE];
1051 		binsheadfree(bp, bufq);
1052 	} else  {
1053 		/*
1054 		 * It has valid data.  Put it on the end of the appropriate
1055 		 * queue, so that it'll stick around for as long as possible.
1056 		 * If buf is AGE, but has dependencies, must put it on last
1057 		 * bufqueue to be scanned, ie LRU. This protects against the
1058 		 * livelock where BQ_AGE only has buffers with dependencies,
1059 		 * and we thus never get to the dependent buffers in BQ_LRU.
1060 		 */
1061 		if (ISSET(bp->b_flags, B_LOCKED)) {
1062 			/* locked in core */
1063 			bufq = &bufqueues[BQ_LOCKED];
1064 		} else if (!ISSET(bp->b_cflags, BC_AGE)) {
1065 			/* valid data */
1066 			bufq = &bufqueues[BQ_LRU];
1067 		} else {
1068 			/* stale but valid data */
1069 			bufq = &bufqueues[BQ_AGE];
1070 		}
1071 		binstailfree(bp, bufq);
1072 	}
1073 already_queued:
1074 	/* Unlock the buffer. */
1075 	CLR(bp->b_cflags, BC_AGE|BC_BUSY|BC_NOCACHE);
1076 	CLR(bp->b_flags, B_ASYNC);
1077 	cv_broadcast(&bp->b_busy);
1078 
1079 	if (bp->b_bufsize <= 0)
1080 		brele(bp);
1081 }
1082 
1083 void
1084 brelse(buf_t *bp, int set)
1085 {
1086 
1087 	mutex_enter(&bufcache_lock);
1088 	brelsel(bp, set);
1089 	mutex_exit(&bufcache_lock);
1090 }
1091 
1092 /*
1093  * Determine if a block is in the cache.
1094  * Just look on what would be its hash chain.  If it's there, return
1095  * a pointer to it, unless it's marked invalid.  If it's marked invalid,
1096  * we normally don't return the buffer, unless the caller explicitly
1097  * wants us to.
1098  */
1099 buf_t *
1100 incore(struct vnode *vp, daddr_t blkno)
1101 {
1102 	buf_t *bp;
1103 
1104 	KASSERT(mutex_owned(&bufcache_lock));
1105 
1106 	/* Search hash chain */
1107 	LIST_FOREACH(bp, BUFHASH(vp, blkno), b_hash) {
1108 		if (bp->b_lblkno == blkno && bp->b_vp == vp &&
1109 		    !ISSET(bp->b_cflags, BC_INVAL)) {
1110 		    	KASSERT(bp->b_objlock == vp->v_interlock);
1111 		    	return (bp);
1112 		}
1113 	}
1114 
1115 	return (NULL);
1116 }
1117 
1118 /*
1119  * Get a block of requested size that is associated with
1120  * a given vnode and block offset. If it is found in the
1121  * block cache, mark it as having been found, make it busy
1122  * and return it. Otherwise, return an empty block of the
1123  * correct size. It is up to the caller to insure that the
1124  * cached blocks be of the correct size.
1125  */
1126 buf_t *
1127 getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, int slptimeo)
1128 {
1129 	int err, preserve;
1130 	buf_t *bp;
1131 
1132 	mutex_enter(&bufcache_lock);
1133  loop:
1134 	bp = incore(vp, blkno);
1135 	if (bp != NULL) {
1136 		err = bbusy(bp, ((slpflag & PCATCH) != 0), slptimeo, NULL);
1137 		if (err != 0) {
1138 			if (err == EPASSTHROUGH)
1139 				goto loop;
1140 			mutex_exit(&bufcache_lock);
1141 			return (NULL);
1142 		}
1143 		KASSERT(!cv_has_waiters(&bp->b_done));
1144 #ifdef DIAGNOSTIC
1145 		if (ISSET(bp->b_oflags, BO_DONE|BO_DELWRI) &&
1146 		    bp->b_bcount < size && vp->v_type != VBLK)
1147 			panic("getblk: block size invariant failed");
1148 #endif
1149 		bremfree(bp);
1150 		preserve = 1;
1151 	} else {
1152 		if ((bp = getnewbuf(slpflag, slptimeo, 0)) == NULL)
1153 			goto loop;
1154 
1155 		if (incore(vp, blkno) != NULL) {
1156 			/* The block has come into memory in the meantime. */
1157 			brelsel(bp, 0);
1158 			goto loop;
1159 		}
1160 
1161 		LIST_INSERT_HEAD(BUFHASH(vp, blkno), bp, b_hash);
1162 		bp->b_blkno = bp->b_lblkno = bp->b_rawblkno = blkno;
1163 		mutex_enter(vp->v_interlock);
1164 		bgetvp(vp, bp);
1165 		mutex_exit(vp->v_interlock);
1166 		preserve = 0;
1167 	}
1168 	mutex_exit(&bufcache_lock);
1169 
1170 	/*
1171 	 * LFS can't track total size of B_LOCKED buffer (locked_queue_bytes)
1172 	 * if we re-size buffers here.
1173 	 */
1174 	if (ISSET(bp->b_flags, B_LOCKED)) {
1175 		KASSERT(bp->b_bufsize >= size);
1176 	} else {
1177 		if (allocbuf(bp, size, preserve)) {
1178 			mutex_enter(&bufcache_lock);
1179 			LIST_REMOVE(bp, b_hash);
1180 			mutex_exit(&bufcache_lock);
1181 			brelse(bp, BC_INVAL);
1182 			return NULL;
1183 		}
1184 	}
1185 	BIO_SETPRIO(bp, BPRIO_DEFAULT);
1186 	return (bp);
1187 }
1188 
1189 /*
1190  * Get an empty, disassociated buffer of given size.
1191  */
1192 buf_t *
1193 geteblk(int size)
1194 {
1195 	buf_t *bp;
1196 	int error;
1197 
1198 	mutex_enter(&bufcache_lock);
1199 	while ((bp = getnewbuf(0, 0, 0)) == NULL)
1200 		;
1201 
1202 	SET(bp->b_cflags, BC_INVAL);
1203 	LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1204 	mutex_exit(&bufcache_lock);
1205 	BIO_SETPRIO(bp, BPRIO_DEFAULT);
1206 	error = allocbuf(bp, size, 0);
1207 	KASSERT(error == 0);
1208 	return (bp);
1209 }
1210 
1211 /*
1212  * Expand or contract the actual memory allocated to a buffer.
1213  *
1214  * If the buffer shrinks, data is lost, so it's up to the
1215  * caller to have written it out *first*; this routine will not
1216  * start a write.  If the buffer grows, it's the callers
1217  * responsibility to fill out the buffer's additional contents.
1218  */
1219 int
1220 allocbuf(buf_t *bp, int size, int preserve)
1221 {
1222 	void *addr;
1223 	vsize_t oldsize, desired_size;
1224 	int oldcount;
1225 	int delta;
1226 
1227 	desired_size = buf_roundsize(size);
1228 	if (desired_size > MAXBSIZE)
1229 		printf("allocbuf: buffer larger than MAXBSIZE requested");
1230 
1231 	oldcount = bp->b_bcount;
1232 
1233 	bp->b_bcount = size;
1234 
1235 	oldsize = bp->b_bufsize;
1236 	if (oldsize == desired_size) {
1237 		/*
1238 		 * Do not short cut the WAPBL resize, as the buffer length
1239 		 * could still have changed and this would corrupt the
1240 		 * tracking of the transaction length.
1241 		 */
1242 		goto out;
1243 	}
1244 
1245 	/*
1246 	 * If we want a buffer of a different size, re-allocate the
1247 	 * buffer's memory; copy old content only if needed.
1248 	 */
1249 	addr = buf_alloc(desired_size);
1250 	if (addr == NULL)
1251 		return ENOMEM;
1252 	if (preserve)
1253 		memcpy(addr, bp->b_data, MIN(oldsize,desired_size));
1254 	if (bp->b_data != NULL)
1255 		buf_mrelease(bp->b_data, oldsize);
1256 	bp->b_data = addr;
1257 	bp->b_bufsize = desired_size;
1258 
1259 	/*
1260 	 * Update overall buffer memory counter (protected by bufcache_lock)
1261 	 */
1262 	delta = (long)desired_size - (long)oldsize;
1263 
1264 	mutex_enter(&bufcache_lock);
1265 	if ((bufmem += delta) > bufmem_hiwater) {
1266 		/*
1267 		 * Need to trim overall memory usage.
1268 		 */
1269 		while (buf_canrelease()) {
1270 			if (curcpu()->ci_schedstate.spc_flags &
1271 			    SPCF_SHOULDYIELD) {
1272 				mutex_exit(&bufcache_lock);
1273 				preempt();
1274 				mutex_enter(&bufcache_lock);
1275 			}
1276 			if (buf_trim() == 0)
1277 				break;
1278 		}
1279 	}
1280 	mutex_exit(&bufcache_lock);
1281 
1282  out:
1283 	if (wapbl_vphaswapbl(bp->b_vp))
1284 		WAPBL_RESIZE_BUF(wapbl_vptomp(bp->b_vp), bp, oldsize, oldcount);
1285 
1286 	return 0;
1287 }
1288 
1289 /*
1290  * Find a buffer which is available for use.
1291  * Select something from a free list.
1292  * Preference is to AGE list, then LRU list.
1293  *
1294  * Called with the buffer queues locked.
1295  * Return buffer locked.
1296  */
1297 buf_t *
1298 getnewbuf(int slpflag, int slptimeo, int from_bufq)
1299 {
1300 	buf_t *bp;
1301 	struct vnode *vp;
1302 
1303  start:
1304 	KASSERT(mutex_owned(&bufcache_lock));
1305 
1306 	/*
1307 	 * Get a new buffer from the pool.
1308 	 */
1309 	if (!from_bufq && buf_lotsfree()) {
1310 		mutex_exit(&bufcache_lock);
1311 		bp = pool_cache_get(buf_cache, PR_NOWAIT);
1312 		if (bp != NULL) {
1313 			memset((char *)bp, 0, sizeof(*bp));
1314 			buf_init(bp);
1315 			SET(bp->b_cflags, BC_BUSY);	/* mark buffer busy */
1316 			mutex_enter(&bufcache_lock);
1317 #if defined(DIAGNOSTIC)
1318 			bp->b_freelistindex = -1;
1319 #endif /* defined(DIAGNOSTIC) */
1320 			return (bp);
1321 		}
1322 		mutex_enter(&bufcache_lock);
1323 	}
1324 
1325 	KASSERT(mutex_owned(&bufcache_lock));
1326 	if ((bp = TAILQ_FIRST(&bufqueues[BQ_AGE].bq_queue)) != NULL ||
1327 	    (bp = TAILQ_FIRST(&bufqueues[BQ_LRU].bq_queue)) != NULL) {
1328 	    	KASSERT(!ISSET(bp->b_cflags, BC_BUSY) || ISSET(bp->b_cflags, BC_VFLUSH));
1329 		bremfree(bp);
1330 
1331 		/* Buffer is no longer on free lists. */
1332 		SET(bp->b_cflags, BC_BUSY);
1333 	} else {
1334 		/*
1335 		 * XXX: !from_bufq should be removed.
1336 		 */
1337 		if (!from_bufq || curlwp != uvm.pagedaemon_lwp) {
1338 			/* wait for a free buffer of any kind */
1339 			if ((slpflag & PCATCH) != 0)
1340 				(void)cv_timedwait_sig(&needbuffer_cv,
1341 				    &bufcache_lock, slptimeo);
1342 			else
1343 				(void)cv_timedwait(&needbuffer_cv,
1344 				    &bufcache_lock, slptimeo);
1345 		}
1346 		return (NULL);
1347 	}
1348 
1349 #ifdef DIAGNOSTIC
1350 	if (bp->b_bufsize <= 0)
1351 		panic("buffer %p: on queue but empty", bp);
1352 #endif
1353 
1354 	if (ISSET(bp->b_cflags, BC_VFLUSH)) {
1355 		/*
1356 		 * This is a delayed write buffer being flushed to disk.  Make
1357 		 * sure it gets aged out of the queue when it's finished, and
1358 		 * leave it off the LRU queue.
1359 		 */
1360 		CLR(bp->b_cflags, BC_VFLUSH);
1361 		SET(bp->b_cflags, BC_AGE);
1362 		goto start;
1363 	}
1364 
1365 	KASSERT(ISSET(bp->b_cflags, BC_BUSY));
1366 	KASSERT(bp->b_refcnt > 0);
1367     	KASSERT(!cv_has_waiters(&bp->b_done));
1368 
1369 	/*
1370 	 * If buffer was a delayed write, start it and return NULL
1371 	 * (since we might sleep while starting the write).
1372 	 */
1373 	if (ISSET(bp->b_oflags, BO_DELWRI)) {
1374 		/*
1375 		 * This buffer has gone through the LRU, so make sure it gets
1376 		 * reused ASAP.
1377 		 */
1378 		SET(bp->b_cflags, BC_AGE);
1379 		mutex_exit(&bufcache_lock);
1380 		bawrite(bp);
1381 		mutex_enter(&bufcache_lock);
1382 		return (NULL);
1383 	}
1384 
1385 	vp = bp->b_vp;
1386 
1387 	/* clear out various other fields */
1388 	bp->b_cflags = BC_BUSY;
1389 	bp->b_oflags = 0;
1390 	bp->b_flags = 0;
1391 	bp->b_dev = NODEV;
1392 	bp->b_blkno = 0;
1393 	bp->b_lblkno = 0;
1394 	bp->b_rawblkno = 0;
1395 	bp->b_iodone = 0;
1396 	bp->b_error = 0;
1397 	bp->b_resid = 0;
1398 	bp->b_bcount = 0;
1399 
1400 	LIST_REMOVE(bp, b_hash);
1401 
1402 	/* Disassociate us from our vnode, if we had one... */
1403 	if (vp != NULL) {
1404 		mutex_enter(vp->v_interlock);
1405 		brelvp(bp);
1406 		mutex_exit(vp->v_interlock);
1407 	}
1408 
1409 	return (bp);
1410 }
1411 
1412 /*
1413  * Attempt to free an aged buffer off the queues.
1414  * Called with queue lock held.
1415  * Returns the amount of buffer memory freed.
1416  */
1417 static int
1418 buf_trim(void)
1419 {
1420 	buf_t *bp;
1421 	long size = 0;
1422 
1423 	KASSERT(mutex_owned(&bufcache_lock));
1424 
1425 	/* Instruct getnewbuf() to get buffers off the queues */
1426 	if ((bp = getnewbuf(PCATCH, 1, 1)) == NULL)
1427 		return 0;
1428 
1429 	KASSERT((bp->b_cflags & BC_WANTED) == 0);
1430 	size = bp->b_bufsize;
1431 	bufmem -= size;
1432 	if (size > 0) {
1433 		buf_mrelease(bp->b_data, size);
1434 		bp->b_bcount = bp->b_bufsize = 0;
1435 	}
1436 	/* brelse() will return the buffer to the global buffer pool */
1437 	brelsel(bp, 0);
1438 	return size;
1439 }
1440 
1441 int
1442 buf_drain(int n)
1443 {
1444 	int size = 0, sz;
1445 
1446 	KASSERT(mutex_owned(&bufcache_lock));
1447 
1448 	while (size < n && bufmem > bufmem_lowater) {
1449 		sz = buf_trim();
1450 		if (sz <= 0)
1451 			break;
1452 		size += sz;
1453 	}
1454 
1455 	return size;
1456 }
1457 
1458 /*
1459  * Wait for operations on the buffer to complete.
1460  * When they do, extract and return the I/O's error value.
1461  */
1462 int
1463 biowait(buf_t *bp)
1464 {
1465 
1466 	KASSERT(ISSET(bp->b_cflags, BC_BUSY));
1467 	KASSERT(bp->b_refcnt > 0);
1468 
1469 	mutex_enter(bp->b_objlock);
1470 	while (!ISSET(bp->b_oflags, BO_DONE | BO_DELWRI))
1471 		cv_wait(&bp->b_done, bp->b_objlock);
1472 	mutex_exit(bp->b_objlock);
1473 
1474 	return bp->b_error;
1475 }
1476 
1477 /*
1478  * Mark I/O complete on a buffer.
1479  *
1480  * If a callback has been requested, e.g. the pageout
1481  * daemon, do so. Otherwise, awaken waiting processes.
1482  *
1483  * [ Leffler, et al., says on p.247:
1484  *	"This routine wakes up the blocked process, frees the buffer
1485  *	for an asynchronous write, or, for a request by the pagedaemon
1486  *	process, invokes a procedure specified in the buffer structure" ]
1487  *
1488  * In real life, the pagedaemon (or other system processes) wants
1489  * to do async stuff to, and doesn't want the buffer brelse()'d.
1490  * (for swap pager, that puts swap buffers on the free lists (!!!),
1491  * for the vn device, that puts allocated buffers on the free lists!)
1492  */
1493 void
1494 biodone(buf_t *bp)
1495 {
1496 	int s;
1497 
1498 	KASSERT(!ISSET(bp->b_oflags, BO_DONE));
1499 
1500 	if (cpu_intr_p()) {
1501 		/* From interrupt mode: defer to a soft interrupt. */
1502 		s = splvm();
1503 		TAILQ_INSERT_TAIL(&curcpu()->ci_data.cpu_biodone, bp, b_actq);
1504 		softint_schedule(biodone_sih);
1505 		splx(s);
1506 	} else {
1507 		/* Process now - the buffer may be freed soon. */
1508 		biodone2(bp);
1509 	}
1510 }
1511 
1512 static void
1513 biodone2(buf_t *bp)
1514 {
1515 	void (*callout)(buf_t *);
1516 
1517 	mutex_enter(bp->b_objlock);
1518 	/* Note that the transfer is done. */
1519 	if (ISSET(bp->b_oflags, BO_DONE))
1520 		panic("biodone2 already");
1521 	CLR(bp->b_flags, B_COWDONE);
1522 	SET(bp->b_oflags, BO_DONE);
1523 	BIO_SETPRIO(bp, BPRIO_DEFAULT);
1524 
1525 	/* Wake up waiting writers. */
1526 	if (!ISSET(bp->b_flags, B_READ))
1527 		vwakeup(bp);
1528 
1529 	if ((callout = bp->b_iodone) != NULL) {
1530 		/* Note callout done, then call out. */
1531 		KASSERT(!cv_has_waiters(&bp->b_done));
1532 		KERNEL_LOCK(1, NULL);		/* XXXSMP */
1533 		bp->b_iodone = NULL;
1534 		mutex_exit(bp->b_objlock);
1535 		(*callout)(bp);
1536 		KERNEL_UNLOCK_ONE(NULL);	/* XXXSMP */
1537 	} else if (ISSET(bp->b_flags, B_ASYNC)) {
1538 		/* If async, release. */
1539 		KASSERT(!cv_has_waiters(&bp->b_done));
1540 		mutex_exit(bp->b_objlock);
1541 		brelse(bp, 0);
1542 	} else {
1543 		/* Otherwise just wake up waiters in biowait(). */
1544 		cv_broadcast(&bp->b_done);
1545 		mutex_exit(bp->b_objlock);
1546 	}
1547 }
1548 
1549 static void
1550 biointr(void *cookie)
1551 {
1552 	struct cpu_info *ci;
1553 	buf_t *bp;
1554 	int s;
1555 
1556 	ci = curcpu();
1557 
1558 	while (!TAILQ_EMPTY(&ci->ci_data.cpu_biodone)) {
1559 		KASSERT(curcpu() == ci);
1560 
1561 		s = splvm();
1562 		bp = TAILQ_FIRST(&ci->ci_data.cpu_biodone);
1563 		TAILQ_REMOVE(&ci->ci_data.cpu_biodone, bp, b_actq);
1564 		splx(s);
1565 
1566 		biodone2(bp);
1567 	}
1568 }
1569 
1570 /*
1571  * Wait for all buffers to complete I/O
1572  * Return the number of "stuck" buffers.
1573  */
1574 int
1575 buf_syncwait(void)
1576 {
1577 	buf_t *bp;
1578 	int iter, nbusy, nbusy_prev = 0, dcount, ihash;
1579 
1580 	dcount = 10000;
1581 	for (iter = 0; iter < 20;) {
1582 		mutex_enter(&bufcache_lock);
1583 		nbusy = 0;
1584 		for (ihash = 0; ihash < bufhash+1; ihash++) {
1585 		    LIST_FOREACH(bp, &bufhashtbl[ihash], b_hash) {
1586 			if ((bp->b_cflags & (BC_BUSY|BC_INVAL)) == BC_BUSY)
1587 				nbusy += ((bp->b_flags & B_READ) == 0);
1588 		    }
1589 		}
1590 		mutex_exit(&bufcache_lock);
1591 
1592 		if (nbusy == 0)
1593 			break;
1594 		if (nbusy_prev == 0)
1595 			nbusy_prev = nbusy;
1596 		printf("%d ", nbusy);
1597 		kpause("bflush", false, MAX(1, hz / 25 * iter), NULL);
1598 		if (nbusy >= nbusy_prev) /* we didn't flush anything */
1599 			iter++;
1600 		else
1601 			nbusy_prev = nbusy;
1602 	}
1603 
1604 	if (nbusy) {
1605 #if defined(DEBUG) || defined(DEBUG_HALT_BUSY)
1606 		printf("giving up\nPrinting vnodes for busy buffers\n");
1607 		for (ihash = 0; ihash < bufhash+1; ihash++) {
1608 		    LIST_FOREACH(bp, &bufhashtbl[ihash], b_hash) {
1609 			if ((bp->b_cflags & (BC_BUSY|BC_INVAL)) == BC_BUSY &&
1610 			    (bp->b_flags & B_READ) == 0)
1611 				vprint(NULL, bp->b_vp);
1612 		    }
1613 		}
1614 #endif
1615 	}
1616 
1617 	return nbusy;
1618 }
1619 
1620 static void
1621 sysctl_fillbuf(buf_t *i, struct buf_sysctl *o)
1622 {
1623 
1624 	o->b_flags = i->b_flags | i->b_cflags | i->b_oflags;
1625 	o->b_error = i->b_error;
1626 	o->b_prio = i->b_prio;
1627 	o->b_dev = i->b_dev;
1628 	o->b_bufsize = i->b_bufsize;
1629 	o->b_bcount = i->b_bcount;
1630 	o->b_resid = i->b_resid;
1631 	o->b_addr = PTRTOUINT64(i->b_data);
1632 	o->b_blkno = i->b_blkno;
1633 	o->b_rawblkno = i->b_rawblkno;
1634 	o->b_iodone = PTRTOUINT64(i->b_iodone);
1635 	o->b_proc = PTRTOUINT64(i->b_proc);
1636 	o->b_vp = PTRTOUINT64(i->b_vp);
1637 	o->b_saveaddr = PTRTOUINT64(i->b_saveaddr);
1638 	o->b_lblkno = i->b_lblkno;
1639 }
1640 
1641 #define KERN_BUFSLOP 20
1642 static int
1643 sysctl_dobuf(SYSCTLFN_ARGS)
1644 {
1645 	buf_t *bp;
1646 	struct buf_sysctl bs;
1647 	struct bqueue *bq;
1648 	char *dp;
1649 	u_int i, op, arg;
1650 	size_t len, needed, elem_size, out_size;
1651 	int error, elem_count, retries;
1652 
1653 	if (namelen == 1 && name[0] == CTL_QUERY)
1654 		return (sysctl_query(SYSCTLFN_CALL(rnode)));
1655 
1656 	if (namelen != 4)
1657 		return (EINVAL);
1658 
1659 	retries = 100;
1660  retry:
1661 	dp = oldp;
1662 	len = (oldp != NULL) ? *oldlenp : 0;
1663 	op = name[0];
1664 	arg = name[1];
1665 	elem_size = name[2];
1666 	elem_count = name[3];
1667 	out_size = MIN(sizeof(bs), elem_size);
1668 
1669 	/*
1670 	 * at the moment, these are just "placeholders" to make the
1671 	 * API for retrieving kern.buf data more extensible in the
1672 	 * future.
1673 	 *
1674 	 * XXX kern.buf currently has "netbsd32" issues.  hopefully
1675 	 * these will be resolved at a later point.
1676 	 */
1677 	if (op != KERN_BUF_ALL || arg != KERN_BUF_ALL ||
1678 	    elem_size < 1 || elem_count < 0)
1679 		return (EINVAL);
1680 
1681 	error = 0;
1682 	needed = 0;
1683 	sysctl_unlock();
1684 	mutex_enter(&bufcache_lock);
1685 	for (i = 0; i < BQUEUES; i++) {
1686 		bq = &bufqueues[i];
1687 		TAILQ_FOREACH(bp, &bq->bq_queue, b_freelist) {
1688 			bq->bq_marker = bp;
1689 			if (len >= elem_size && elem_count > 0) {
1690 				sysctl_fillbuf(bp, &bs);
1691 				mutex_exit(&bufcache_lock);
1692 				error = copyout(&bs, dp, out_size);
1693 				mutex_enter(&bufcache_lock);
1694 				if (error)
1695 					break;
1696 				if (bq->bq_marker != bp) {
1697 					/*
1698 					 * This sysctl node is only for
1699 					 * statistics.  Retry; if the
1700 					 * queue keeps changing, then
1701 					 * bail out.
1702 					 */
1703 					if (retries-- == 0) {
1704 						error = EAGAIN;
1705 						break;
1706 					}
1707 					mutex_exit(&bufcache_lock);
1708 					goto retry;
1709 				}
1710 				dp += elem_size;
1711 				len -= elem_size;
1712 			}
1713 			needed += elem_size;
1714 			if (elem_count > 0 && elem_count != INT_MAX)
1715 				elem_count--;
1716 		}
1717 		if (error != 0)
1718 			break;
1719 	}
1720 	mutex_exit(&bufcache_lock);
1721 	sysctl_relock();
1722 
1723 	*oldlenp = needed;
1724 	if (oldp == NULL)
1725 		*oldlenp += KERN_BUFSLOP * sizeof(buf_t);
1726 
1727 	return (error);
1728 }
1729 
1730 static int
1731 sysctl_bufvm_update(SYSCTLFN_ARGS)
1732 {
1733 	int t, error, rv;
1734 	struct sysctlnode node;
1735 
1736 	node = *rnode;
1737 	node.sysctl_data = &t;
1738 	t = *(int *)rnode->sysctl_data;
1739 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
1740 	if (error || newp == NULL)
1741 		return (error);
1742 
1743 	if (t < 0)
1744 		return EINVAL;
1745 	if (rnode->sysctl_data == &bufcache) {
1746 		if (t > 100)
1747 			return (EINVAL);
1748 		bufcache = t;
1749 		buf_setwm();
1750 	} else if (rnode->sysctl_data == &bufmem_lowater) {
1751 		if (bufmem_hiwater - t < 16)
1752 			return (EINVAL);
1753 		bufmem_lowater = t;
1754 	} else if (rnode->sysctl_data == &bufmem_hiwater) {
1755 		if (t - bufmem_lowater < 16)
1756 			return (EINVAL);
1757 		bufmem_hiwater = t;
1758 	} else
1759 		return (EINVAL);
1760 
1761 	/* Drain until below new high water mark */
1762 	sysctl_unlock();
1763 	mutex_enter(&bufcache_lock);
1764 	while ((t = bufmem - bufmem_hiwater) >= 0) {
1765 		rv = buf_drain(t / (2 * 1024));
1766 		if (rv <= 0)
1767 			break;
1768 	}
1769 	mutex_exit(&bufcache_lock);
1770 	sysctl_relock();
1771 
1772 	return 0;
1773 }
1774 
1775 static struct sysctllog *vfsbio_sysctllog;
1776 
1777 static void
1778 sysctl_kern_buf_setup(void)
1779 {
1780 
1781 	sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1782 		       CTLFLAG_PERMANENT,
1783 		       CTLTYPE_NODE, "kern", NULL,
1784 		       NULL, 0, NULL, 0,
1785 		       CTL_KERN, CTL_EOL);
1786 	sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1787 		       CTLFLAG_PERMANENT,
1788 		       CTLTYPE_NODE, "buf",
1789 		       SYSCTL_DESCR("Kernel buffer cache information"),
1790 		       sysctl_dobuf, 0, NULL, 0,
1791 		       CTL_KERN, KERN_BUF, CTL_EOL);
1792 }
1793 
1794 static void
1795 sysctl_vm_buf_setup(void)
1796 {
1797 
1798 	sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1799 		       CTLFLAG_PERMANENT,
1800 		       CTLTYPE_NODE, "vm", NULL,
1801 		       NULL, 0, NULL, 0,
1802 		       CTL_VM, CTL_EOL);
1803 	sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1804 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1805 		       CTLTYPE_INT, "bufcache",
1806 		       SYSCTL_DESCR("Percentage of physical memory to use for "
1807 				    "buffer cache"),
1808 		       sysctl_bufvm_update, 0, &bufcache, 0,
1809 		       CTL_VM, CTL_CREATE, CTL_EOL);
1810 	sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1811 		       CTLFLAG_PERMANENT|CTLFLAG_READONLY,
1812 		       CTLTYPE_INT, "bufmem",
1813 		       SYSCTL_DESCR("Amount of kernel memory used by buffer "
1814 				    "cache"),
1815 		       NULL, 0, &bufmem, 0,
1816 		       CTL_VM, CTL_CREATE, CTL_EOL);
1817 	sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1818 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1819 		       CTLTYPE_INT, "bufmem_lowater",
1820 		       SYSCTL_DESCR("Minimum amount of kernel memory to "
1821 				    "reserve for buffer cache"),
1822 		       sysctl_bufvm_update, 0, &bufmem_lowater, 0,
1823 		       CTL_VM, CTL_CREATE, CTL_EOL);
1824 	sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1825 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1826 		       CTLTYPE_INT, "bufmem_hiwater",
1827 		       SYSCTL_DESCR("Maximum amount of kernel memory to use "
1828 				    "for buffer cache"),
1829 		       sysctl_bufvm_update, 0, &bufmem_hiwater, 0,
1830 		       CTL_VM, CTL_CREATE, CTL_EOL);
1831 }
1832 
1833 #ifdef DEBUG
1834 /*
1835  * Print out statistics on the current allocation of the buffer pool.
1836  * Can be enabled to print out on every ``sync'' by setting "syncprt"
1837  * in vfs_syscalls.c using sysctl.
1838  */
1839 void
1840 vfs_bufstats(void)
1841 {
1842 	int i, j, count;
1843 	buf_t *bp;
1844 	struct bqueue *dp;
1845 	int counts[(MAXBSIZE / PAGE_SIZE) + 1];
1846 	static const char *bname[BQUEUES] = { "LOCKED", "LRU", "AGE" };
1847 
1848 	for (dp = bufqueues, i = 0; dp < &bufqueues[BQUEUES]; dp++, i++) {
1849 		count = 0;
1850 		for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++)
1851 			counts[j] = 0;
1852 		TAILQ_FOREACH(bp, &dp->bq_queue, b_freelist) {
1853 			counts[bp->b_bufsize/PAGE_SIZE]++;
1854 			count++;
1855 		}
1856 		printf("%s: total-%d", bname[i], count);
1857 		for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++)
1858 			if (counts[j] != 0)
1859 				printf(", %d-%d", j * PAGE_SIZE, counts[j]);
1860 		printf("\n");
1861 	}
1862 }
1863 #endif /* DEBUG */
1864 
1865 /* ------------------------------ */
1866 
1867 buf_t *
1868 getiobuf(struct vnode *vp, bool waitok)
1869 {
1870 	buf_t *bp;
1871 
1872 	bp = pool_cache_get(bufio_cache, (waitok ? PR_WAITOK : PR_NOWAIT));
1873 	if (bp == NULL)
1874 		return bp;
1875 
1876 	buf_init(bp);
1877 
1878 	if ((bp->b_vp = vp) == NULL)
1879 		bp->b_objlock = &buffer_lock;
1880 	else
1881 		bp->b_objlock = vp->v_interlock;
1882 
1883 	return bp;
1884 }
1885 
1886 void
1887 putiobuf(buf_t *bp)
1888 {
1889 
1890 	buf_destroy(bp);
1891 	pool_cache_put(bufio_cache, bp);
1892 }
1893 
1894 /*
1895  * nestiobuf_iodone: b_iodone callback for nested buffers.
1896  */
1897 
1898 void
1899 nestiobuf_iodone(buf_t *bp)
1900 {
1901 	buf_t *mbp = bp->b_private;
1902 	int error;
1903 	int donebytes;
1904 
1905 	KASSERT(bp->b_bcount <= bp->b_bufsize);
1906 	KASSERT(mbp != bp);
1907 
1908 	error = bp->b_error;
1909 	if (bp->b_error == 0 &&
1910 	    (bp->b_bcount < bp->b_bufsize || bp->b_resid > 0)) {
1911 		/*
1912 		 * Not all got transfered, raise an error. We have no way to
1913 		 * propagate these conditions to mbp.
1914 		 */
1915 		error = EIO;
1916 	}
1917 
1918 	donebytes = bp->b_bufsize;
1919 
1920 	putiobuf(bp);
1921 	nestiobuf_done(mbp, donebytes, error);
1922 }
1923 
1924 /*
1925  * nestiobuf_setup: setup a "nested" buffer.
1926  *
1927  * => 'mbp' is a "master" buffer which is being divided into sub pieces.
1928  * => 'bp' should be a buffer allocated by getiobuf.
1929  * => 'offset' is a byte offset in the master buffer.
1930  * => 'size' is a size in bytes of this nested buffer.
1931  */
1932 
1933 void
1934 nestiobuf_setup(buf_t *mbp, buf_t *bp, int offset, size_t size)
1935 {
1936 	const int b_read = mbp->b_flags & B_READ;
1937 	struct vnode *vp = mbp->b_vp;
1938 
1939 	KASSERT(mbp->b_bcount >= offset + size);
1940 	bp->b_vp = vp;
1941 	bp->b_dev = mbp->b_dev;
1942 	bp->b_objlock = mbp->b_objlock;
1943 	bp->b_cflags = BC_BUSY;
1944 	bp->b_flags = B_ASYNC | b_read;
1945 	bp->b_iodone = nestiobuf_iodone;
1946 	bp->b_data = (char *)mbp->b_data + offset;
1947 	bp->b_resid = bp->b_bcount = size;
1948 	bp->b_bufsize = bp->b_bcount;
1949 	bp->b_private = mbp;
1950 	BIO_COPYPRIO(bp, mbp);
1951 	if (!b_read && vp != NULL) {
1952 		mutex_enter(vp->v_interlock);
1953 		vp->v_numoutput++;
1954 		mutex_exit(vp->v_interlock);
1955 	}
1956 }
1957 
1958 /*
1959  * nestiobuf_done: propagate completion to the master buffer.
1960  *
1961  * => 'donebytes' specifies how many bytes in the 'mbp' is completed.
1962  * => 'error' is an errno(2) that 'donebytes' has been completed with.
1963  */
1964 
1965 void
1966 nestiobuf_done(buf_t *mbp, int donebytes, int error)
1967 {
1968 
1969 	if (donebytes == 0) {
1970 		return;
1971 	}
1972 	mutex_enter(mbp->b_objlock);
1973 	KASSERT(mbp->b_resid >= donebytes);
1974 	mbp->b_resid -= donebytes;
1975 	if (error)
1976 		mbp->b_error = error;
1977 	if (mbp->b_resid == 0) {
1978 		if (mbp->b_error)
1979 			mbp->b_resid = mbp->b_bcount;
1980 		mutex_exit(mbp->b_objlock);
1981 		biodone(mbp);
1982 	} else
1983 		mutex_exit(mbp->b_objlock);
1984 }
1985 
1986 void
1987 buf_init(buf_t *bp)
1988 {
1989 
1990 	cv_init(&bp->b_busy, "biolock");
1991 	cv_init(&bp->b_done, "biowait");
1992 	bp->b_dev = NODEV;
1993 	bp->b_error = 0;
1994 	bp->b_flags = 0;
1995 	bp->b_cflags = 0;
1996 	bp->b_oflags = 0;
1997 	bp->b_objlock = &buffer_lock;
1998 	bp->b_iodone = NULL;
1999 	bp->b_refcnt = 1;
2000 	bp->b_dev = NODEV;
2001 	bp->b_vnbufs.le_next = NOLIST;
2002 	BIO_SETPRIO(bp, BPRIO_DEFAULT);
2003 }
2004 
2005 void
2006 buf_destroy(buf_t *bp)
2007 {
2008 
2009 	cv_destroy(&bp->b_done);
2010 	cv_destroy(&bp->b_busy);
2011 }
2012 
2013 int
2014 bbusy(buf_t *bp, bool intr, int timo, kmutex_t *interlock)
2015 {
2016 	int error;
2017 
2018 	KASSERT(mutex_owned(&bufcache_lock));
2019 
2020 	if ((bp->b_cflags & BC_BUSY) != 0) {
2021 		if (curlwp == uvm.pagedaemon_lwp)
2022 			return EDEADLK;
2023 		bp->b_cflags |= BC_WANTED;
2024 		bref(bp);
2025 		if (interlock != NULL)
2026 			mutex_exit(interlock);
2027 		if (intr) {
2028 			error = cv_timedwait_sig(&bp->b_busy, &bufcache_lock,
2029 			    timo);
2030 		} else {
2031 			error = cv_timedwait(&bp->b_busy, &bufcache_lock,
2032 			    timo);
2033 		}
2034 		brele(bp);
2035 		if (interlock != NULL)
2036 			mutex_enter(interlock);
2037 		if (error != 0)
2038 			return error;
2039 		return EPASSTHROUGH;
2040 	}
2041 	bp->b_cflags |= BC_BUSY;
2042 
2043 	return 0;
2044 }
2045