xref: /netbsd-src/sys/kern/vfs_bio.c (revision f89f6560d453f5e37386cc7938c072d2f528b9fa)
1 /*	$NetBSD: vfs_bio.c,v 1.255 2015/03/28 19:24:04 maxv Exp $	*/
2 
3 /*-
4  * Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Andrew Doran, and by Wasabi Systems, Inc.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 /*-
33  * Copyright (c) 1982, 1986, 1989, 1993
34  *	The Regents of the University of California.  All rights reserved.
35  * (c) UNIX System Laboratories, Inc.
36  * All or some portions of this file are derived from material licensed
37  * to the University of California by American Telephone and Telegraph
38  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
39  * the permission of UNIX System Laboratories, Inc.
40  *
41  * Redistribution and use in source and binary forms, with or without
42  * modification, are permitted provided that the following conditions
43  * are met:
44  * 1. Redistributions of source code must retain the above copyright
45  *    notice, this list of conditions and the following disclaimer.
46  * 2. Redistributions in binary form must reproduce the above copyright
47  *    notice, this list of conditions and the following disclaimer in the
48  *    documentation and/or other materials provided with the distribution.
49  * 3. Neither the name of the University nor the names of its contributors
50  *    may be used to endorse or promote products derived from this software
51  *    without specific prior written permission.
52  *
53  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
54  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
55  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
56  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
57  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
58  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
59  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
60  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
61  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
62  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
63  * SUCH DAMAGE.
64  *
65  *	@(#)vfs_bio.c	8.6 (Berkeley) 1/11/94
66  */
67 
68 /*-
69  * Copyright (c) 1994 Christopher G. Demetriou
70  *
71  * Redistribution and use in source and binary forms, with or without
72  * modification, are permitted provided that the following conditions
73  * are met:
74  * 1. Redistributions of source code must retain the above copyright
75  *    notice, this list of conditions and the following disclaimer.
76  * 2. Redistributions in binary form must reproduce the above copyright
77  *    notice, this list of conditions and the following disclaimer in the
78  *    documentation and/or other materials provided with the distribution.
79  * 3. All advertising materials mentioning features or use of this software
80  *    must display the following acknowledgement:
81  *	This product includes software developed by the University of
82  *	California, Berkeley and its contributors.
83  * 4. Neither the name of the University nor the names of its contributors
84  *    may be used to endorse or promote products derived from this software
85  *    without specific prior written permission.
86  *
87  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
88  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
89  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
90  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
91  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
92  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
93  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
94  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
95  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
96  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
97  * SUCH DAMAGE.
98  *
99  *	@(#)vfs_bio.c	8.6 (Berkeley) 1/11/94
100  */
101 
102 /*
103  * The buffer cache subsystem.
104  *
105  * Some references:
106  *	Bach: The Design of the UNIX Operating System (Prentice Hall, 1986)
107  *	Leffler, et al.: The Design and Implementation of the 4.3BSD
108  *		UNIX Operating System (Addison Welley, 1989)
109  *
110  * Locking
111  *
112  * There are three locks:
113  * - bufcache_lock: protects global buffer cache state.
114  * - BC_BUSY: a long term per-buffer lock.
115  * - buf_t::b_objlock: lock on completion (biowait vs biodone).
116  *
117  * For buffers associated with vnodes (a most common case) b_objlock points
118  * to the vnode_t::v_interlock.  Otherwise, it points to generic buffer_lock.
119  *
120  * Lock order:
121  *	bufcache_lock ->
122  *		buf_t::b_objlock
123  */
124 
125 #include <sys/cdefs.h>
126 __KERNEL_RCSID(0, "$NetBSD: vfs_bio.c,v 1.255 2015/03/28 19:24:04 maxv Exp $");
127 
128 #include "opt_bufcache.h"
129 
130 #include <sys/param.h>
131 #include <sys/systm.h>
132 #include <sys/kernel.h>
133 #include <sys/proc.h>
134 #include <sys/buf.h>
135 #include <sys/vnode.h>
136 #include <sys/mount.h>
137 #include <sys/resourcevar.h>
138 #include <sys/sysctl.h>
139 #include <sys/conf.h>
140 #include <sys/kauth.h>
141 #include <sys/fstrans.h>
142 #include <sys/intr.h>
143 #include <sys/cpu.h>
144 #include <sys/wapbl.h>
145 #include <sys/bitops.h>
146 #include <sys/cprng.h>
147 
148 #include <uvm/uvm.h>	/* extern struct uvm uvm */
149 
150 #include <miscfs/specfs/specdev.h>
151 
152 #ifndef	BUFPAGES
153 # define BUFPAGES 0
154 #endif
155 
156 #ifdef BUFCACHE
157 # if (BUFCACHE < 5) || (BUFCACHE > 95)
158 #  error BUFCACHE is not between 5 and 95
159 # endif
160 #else
161 # define BUFCACHE 15
162 #endif
163 
164 u_int	nbuf;			/* desired number of buffer headers */
165 u_int	bufpages = BUFPAGES;	/* optional hardwired count */
166 u_int	bufcache = BUFCACHE;	/* max % of RAM to use for buffer cache */
167 
168 /* Function prototypes */
169 struct bqueue;
170 
171 static void buf_setwm(void);
172 static int buf_trim(void);
173 static void *bufpool_page_alloc(struct pool *, int);
174 static void bufpool_page_free(struct pool *, void *);
175 static buf_t *bio_doread(struct vnode *, daddr_t, int, int);
176 static buf_t *getnewbuf(int, int, int);
177 static int buf_lotsfree(void);
178 static int buf_canrelease(void);
179 static u_long buf_mempoolidx(u_long);
180 static u_long buf_roundsize(u_long);
181 static void *buf_alloc(size_t);
182 static void buf_mrelease(void *, size_t);
183 static void binsheadfree(buf_t *, struct bqueue *);
184 static void binstailfree(buf_t *, struct bqueue *);
185 #ifdef DEBUG
186 static int checkfreelist(buf_t *, struct bqueue *, int);
187 #endif
188 static void biointr(void *);
189 static void biodone2(buf_t *);
190 static void bref(buf_t *);
191 static void brele(buf_t *);
192 static void sysctl_kern_buf_setup(void);
193 static void sysctl_vm_buf_setup(void);
194 
195 /*
196  * Definitions for the buffer hash lists.
197  */
198 #define	BUFHASH(dvp, lbn)	\
199 	(&bufhashtbl[(((long)(dvp) >> 8) + (int)(lbn)) & bufhash])
200 LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash;
201 u_long	bufhash;
202 struct bqueue bufqueues[BQUEUES];
203 
204 static kcondvar_t needbuffer_cv;
205 
206 /*
207  * Buffer queue lock.
208  */
209 kmutex_t bufcache_lock;
210 kmutex_t buffer_lock;
211 
212 /* Software ISR for completed transfers. */
213 static void *biodone_sih;
214 
215 /* Buffer pool for I/O buffers. */
216 static pool_cache_t buf_cache;
217 static pool_cache_t bufio_cache;
218 
219 #define MEMPOOL_INDEX_OFFSET (ilog2(DEV_BSIZE))	/* smallest pool is 512 bytes */
220 #define NMEMPOOLS (ilog2(MAXBSIZE) - MEMPOOL_INDEX_OFFSET + 1)
221 __CTASSERT((1 << (NMEMPOOLS + MEMPOOL_INDEX_OFFSET - 1)) == MAXBSIZE);
222 
223 /* Buffer memory pools */
224 static struct pool bmempools[NMEMPOOLS];
225 
226 static struct vm_map *buf_map;
227 
228 /*
229  * Buffer memory pool allocator.
230  */
231 static void *
232 bufpool_page_alloc(struct pool *pp, int flags)
233 {
234 
235 	return (void *)uvm_km_alloc(buf_map,
236 	    MAXBSIZE, MAXBSIZE,
237 	    ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT|UVM_KMF_TRYLOCK)
238 	    | UVM_KMF_WIRED);
239 }
240 
241 static void
242 bufpool_page_free(struct pool *pp, void *v)
243 {
244 
245 	uvm_km_free(buf_map, (vaddr_t)v, MAXBSIZE, UVM_KMF_WIRED);
246 }
247 
248 static struct pool_allocator bufmempool_allocator = {
249 	.pa_alloc = bufpool_page_alloc,
250 	.pa_free = bufpool_page_free,
251 	.pa_pagesz = MAXBSIZE,
252 };
253 
254 /* Buffer memory management variables */
255 u_long bufmem_valimit;
256 u_long bufmem_hiwater;
257 u_long bufmem_lowater;
258 u_long bufmem;
259 
260 /*
261  * MD code can call this to set a hard limit on the amount
262  * of virtual memory used by the buffer cache.
263  */
264 int
265 buf_setvalimit(vsize_t sz)
266 {
267 
268 	/* We need to accommodate at least NMEMPOOLS of MAXBSIZE each */
269 	if (sz < NMEMPOOLS * MAXBSIZE)
270 		return EINVAL;
271 
272 	bufmem_valimit = sz;
273 	return 0;
274 }
275 
276 static void
277 buf_setwm(void)
278 {
279 
280 	bufmem_hiwater = buf_memcalc();
281 	/* lowater is approx. 2% of memory (with bufcache = 15) */
282 #define	BUFMEM_WMSHIFT	3
283 #define	BUFMEM_HIWMMIN	(64 * 1024 << BUFMEM_WMSHIFT)
284 	if (bufmem_hiwater < BUFMEM_HIWMMIN)
285 		/* Ensure a reasonable minimum value */
286 		bufmem_hiwater = BUFMEM_HIWMMIN;
287 	bufmem_lowater = bufmem_hiwater >> BUFMEM_WMSHIFT;
288 }
289 
290 #ifdef DEBUG
291 int debug_verify_freelist = 0;
292 static int
293 checkfreelist(buf_t *bp, struct bqueue *dp, int ison)
294 {
295 	buf_t *b;
296 
297 	if (!debug_verify_freelist)
298 		return 1;
299 
300 	TAILQ_FOREACH(b, &dp->bq_queue, b_freelist) {
301 		if (b == bp)
302 			return ison ? 1 : 0;
303 	}
304 
305 	return ison ? 0 : 1;
306 }
307 #endif
308 
309 /*
310  * Insq/Remq for the buffer hash lists.
311  * Call with buffer queue locked.
312  */
313 static void
314 binsheadfree(buf_t *bp, struct bqueue *dp)
315 {
316 
317 	KASSERT(mutex_owned(&bufcache_lock));
318 	KASSERT(bp->b_freelistindex == -1);
319 	TAILQ_INSERT_HEAD(&dp->bq_queue, bp, b_freelist);
320 	dp->bq_bytes += bp->b_bufsize;
321 	bp->b_freelistindex = dp - bufqueues;
322 }
323 
324 static void
325 binstailfree(buf_t *bp, struct bqueue *dp)
326 {
327 
328 	KASSERT(mutex_owned(&bufcache_lock));
329 	KASSERT(bp->b_freelistindex == -1);
330 	TAILQ_INSERT_TAIL(&dp->bq_queue, bp, b_freelist);
331 	dp->bq_bytes += bp->b_bufsize;
332 	bp->b_freelistindex = dp - bufqueues;
333 }
334 
335 void
336 bremfree(buf_t *bp)
337 {
338 	struct bqueue *dp;
339 	int bqidx = bp->b_freelistindex;
340 
341 	KASSERT(mutex_owned(&bufcache_lock));
342 
343 	KASSERT(bqidx != -1);
344 	dp = &bufqueues[bqidx];
345 	KDASSERT(checkfreelist(bp, dp, 1));
346 	KASSERT(dp->bq_bytes >= bp->b_bufsize);
347 	TAILQ_REMOVE(&dp->bq_queue, bp, b_freelist);
348 	dp->bq_bytes -= bp->b_bufsize;
349 
350 	/* For the sysctl helper. */
351 	if (bp == dp->bq_marker)
352 		dp->bq_marker = NULL;
353 
354 #if defined(DIAGNOSTIC)
355 	bp->b_freelistindex = -1;
356 #endif /* defined(DIAGNOSTIC) */
357 }
358 
359 /*
360  * Add a reference to an buffer structure that came from buf_cache.
361  */
362 static inline void
363 bref(buf_t *bp)
364 {
365 
366 	KASSERT(mutex_owned(&bufcache_lock));
367 	KASSERT(bp->b_refcnt > 0);
368 
369 	bp->b_refcnt++;
370 }
371 
372 /*
373  * Free an unused buffer structure that came from buf_cache.
374  */
375 static inline void
376 brele(buf_t *bp)
377 {
378 
379 	KASSERT(mutex_owned(&bufcache_lock));
380 	KASSERT(bp->b_refcnt > 0);
381 
382 	if (bp->b_refcnt-- == 1) {
383 		buf_destroy(bp);
384 #ifdef DEBUG
385 		memset((char *)bp, 0, sizeof(*bp));
386 #endif
387 		pool_cache_put(buf_cache, bp);
388 	}
389 }
390 
391 /*
392  * note that for some ports this is used by pmap bootstrap code to
393  * determine kva size.
394  */
395 u_long
396 buf_memcalc(void)
397 {
398 	u_long n;
399 	vsize_t mapsz = 0;
400 
401 	/*
402 	 * Determine the upper bound of memory to use for buffers.
403 	 *
404 	 *	- If bufpages is specified, use that as the number
405 	 *	  pages.
406 	 *
407 	 *	- Otherwise, use bufcache as the percentage of
408 	 *	  physical memory.
409 	 */
410 	if (bufpages != 0) {
411 		n = bufpages;
412 	} else {
413 		if (bufcache < 5) {
414 			printf("forcing bufcache %d -> 5", bufcache);
415 			bufcache = 5;
416 		}
417 		if (bufcache > 95) {
418 			printf("forcing bufcache %d -> 95", bufcache);
419 			bufcache = 95;
420 		}
421 		if (buf_map != NULL)
422 			mapsz = vm_map_max(buf_map) - vm_map_min(buf_map);
423 		n = calc_cache_size(mapsz, bufcache,
424 		    (buf_map != kernel_map) ? 100 : BUFCACHE_VA_MAXPCT)
425 		    / PAGE_SIZE;
426 	}
427 
428 	n <<= PAGE_SHIFT;
429 	if (bufmem_valimit != 0 && n > bufmem_valimit)
430 		n = bufmem_valimit;
431 
432 	return (n);
433 }
434 
435 /*
436  * Initialize buffers and hash links for buffers.
437  */
438 void
439 bufinit(void)
440 {
441 	struct bqueue *dp;
442 	int use_std;
443 	u_int i;
444 	extern void (*biodone_vfs)(buf_t *);
445 
446 	biodone_vfs = biodone;
447 
448 	mutex_init(&bufcache_lock, MUTEX_DEFAULT, IPL_NONE);
449 	mutex_init(&buffer_lock, MUTEX_DEFAULT, IPL_NONE);
450 	cv_init(&needbuffer_cv, "needbuf");
451 
452 	if (bufmem_valimit != 0) {
453 		vaddr_t minaddr = 0, maxaddr;
454 		buf_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
455 					  bufmem_valimit, 0, false, 0);
456 		if (buf_map == NULL)
457 			panic("bufinit: cannot allocate submap");
458 	} else
459 		buf_map = kernel_map;
460 
461 	/*
462 	 * Initialize buffer cache memory parameters.
463 	 */
464 	bufmem = 0;
465 	buf_setwm();
466 
467 	/* On "small" machines use small pool page sizes where possible */
468 	use_std = (physmem < atop(16*1024*1024));
469 
470 	/*
471 	 * Also use them on systems that can map the pool pages using
472 	 * a direct-mapped segment.
473 	 */
474 #ifdef PMAP_MAP_POOLPAGE
475 	use_std = 1;
476 #endif
477 
478 	buf_cache = pool_cache_init(sizeof(buf_t), 0, 0, 0,
479 	    "bufpl", NULL, IPL_SOFTBIO, NULL, NULL, NULL);
480 	bufio_cache = pool_cache_init(sizeof(buf_t), 0, 0, 0,
481 	    "biopl", NULL, IPL_BIO, NULL, NULL, NULL);
482 
483 	for (i = 0; i < NMEMPOOLS; i++) {
484 		struct pool_allocator *pa;
485 		struct pool *pp = &bmempools[i];
486 		u_int size = 1 << (i + MEMPOOL_INDEX_OFFSET);
487 		char *name = kmem_alloc(8, KM_SLEEP); /* XXX: never freed */
488 		if (__predict_false(size >= 1048576))
489 			(void)snprintf(name, 8, "buf%um", size / 1048576);
490 		else if (__predict_true(size >= 1024))
491 			(void)snprintf(name, 8, "buf%uk", size / 1024);
492 		else
493 			(void)snprintf(name, 8, "buf%ub", size);
494 		pa = (size <= PAGE_SIZE && use_std)
495 			? &pool_allocator_nointr
496 			: &bufmempool_allocator;
497 		pool_init(pp, size, 0, 0, 0, name, pa, IPL_NONE);
498 		pool_setlowat(pp, 1);
499 		pool_sethiwat(pp, 1);
500 	}
501 
502 	/* Initialize the buffer queues */
503 	for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++) {
504 		TAILQ_INIT(&dp->bq_queue);
505 		dp->bq_bytes = 0;
506 	}
507 
508 	/*
509 	 * Estimate hash table size based on the amount of memory we
510 	 * intend to use for the buffer cache. The average buffer
511 	 * size is dependent on our clients (i.e. filesystems).
512 	 *
513 	 * For now, use an empirical 3K per buffer.
514 	 */
515 	nbuf = (bufmem_hiwater / 1024) / 3;
516 	bufhashtbl = hashinit(nbuf, HASH_LIST, true, &bufhash);
517 
518 	sysctl_kern_buf_setup();
519 	sysctl_vm_buf_setup();
520 }
521 
522 void
523 bufinit2(void)
524 {
525 
526 	biodone_sih = softint_establish(SOFTINT_BIO | SOFTINT_MPSAFE, biointr,
527 	    NULL);
528 	if (biodone_sih == NULL)
529 		panic("bufinit2: can't establish soft interrupt");
530 }
531 
532 static int
533 buf_lotsfree(void)
534 {
535 	u_long guess;
536 
537 	/* Always allocate if less than the low water mark. */
538 	if (bufmem < bufmem_lowater)
539 		return 1;
540 
541 	/* Never allocate if greater than the high water mark. */
542 	if (bufmem > bufmem_hiwater)
543 		return 0;
544 
545 	/* If there's anything on the AGE list, it should be eaten. */
546 	if (TAILQ_FIRST(&bufqueues[BQ_AGE].bq_queue) != NULL)
547 		return 0;
548 
549 	/*
550 	 * The probabily of getting a new allocation is inversely
551 	 * proportional  to the current size of the cache above
552 	 * the low water mark.  Divide the total first to avoid overflows
553 	 * in the product.
554 	 */
555 	guess = cprng_fast32() % 16;
556 
557 	if ((bufmem_hiwater - bufmem_lowater) / 16 * guess >=
558 	    (bufmem - bufmem_lowater))
559 		return 1;
560 
561 	/* Otherwise don't allocate. */
562 	return 0;
563 }
564 
565 /*
566  * Return estimate of bytes we think need to be
567  * released to help resolve low memory conditions.
568  *
569  * => called with bufcache_lock held.
570  */
571 static int
572 buf_canrelease(void)
573 {
574 	int pagedemand, ninvalid = 0;
575 
576 	KASSERT(mutex_owned(&bufcache_lock));
577 
578 	if (bufmem < bufmem_lowater)
579 		return 0;
580 
581 	if (bufmem > bufmem_hiwater)
582 		return bufmem - bufmem_hiwater;
583 
584 	ninvalid += bufqueues[BQ_AGE].bq_bytes;
585 
586 	pagedemand = uvmexp.freetarg - uvmexp.free;
587 	if (pagedemand < 0)
588 		return ninvalid;
589 	return MAX(ninvalid, MIN(2 * MAXBSIZE,
590 	    MIN((bufmem - bufmem_lowater) / 16, pagedemand * PAGE_SIZE)));
591 }
592 
593 /*
594  * Buffer memory allocation helper functions
595  */
596 static u_long
597 buf_mempoolidx(u_long size)
598 {
599 	u_int n = 0;
600 
601 	size -= 1;
602 	size >>= MEMPOOL_INDEX_OFFSET;
603 	while (size) {
604 		size >>= 1;
605 		n += 1;
606 	}
607 	if (n >= NMEMPOOLS)
608 		panic("buf mem pool index %d", n);
609 	return n;
610 }
611 
612 static u_long
613 buf_roundsize(u_long size)
614 {
615 	/* Round up to nearest power of 2 */
616 	return (1 << (buf_mempoolidx(size) + MEMPOOL_INDEX_OFFSET));
617 }
618 
619 static void *
620 buf_alloc(size_t size)
621 {
622 	u_int n = buf_mempoolidx(size);
623 	void *addr;
624 
625 	while (1) {
626 		addr = pool_get(&bmempools[n], PR_NOWAIT);
627 		if (addr != NULL)
628 			break;
629 
630 		/* No memory, see if we can free some. If so, try again */
631 		mutex_enter(&bufcache_lock);
632 		if (buf_drain(1) > 0) {
633 			mutex_exit(&bufcache_lock);
634 			continue;
635 		}
636 
637 		if (curlwp == uvm.pagedaemon_lwp) {
638 			mutex_exit(&bufcache_lock);
639 			return NULL;
640 		}
641 
642 		/* Wait for buffers to arrive on the LRU queue */
643 		cv_timedwait(&needbuffer_cv, &bufcache_lock, hz / 4);
644 		mutex_exit(&bufcache_lock);
645 	}
646 
647 	return addr;
648 }
649 
650 static void
651 buf_mrelease(void *addr, size_t size)
652 {
653 
654 	pool_put(&bmempools[buf_mempoolidx(size)], addr);
655 }
656 
657 /*
658  * bread()/breadn() helper.
659  */
660 static buf_t *
661 bio_doread(struct vnode *vp, daddr_t blkno, int size, int async)
662 {
663 	buf_t *bp;
664 	struct mount *mp;
665 
666 	bp = getblk(vp, blkno, size, 0, 0);
667 
668 	/*
669 	 * getblk() may return NULL if we are the pagedaemon.
670 	 */
671 	if (bp == NULL) {
672 		KASSERT(curlwp == uvm.pagedaemon_lwp);
673 		return NULL;
674 	}
675 
676 	/*
677 	 * If buffer does not have data valid, start a read.
678 	 * Note that if buffer is BC_INVAL, getblk() won't return it.
679 	 * Therefore, it's valid if its I/O has completed or been delayed.
680 	 */
681 	if (!ISSET(bp->b_oflags, (BO_DONE | BO_DELWRI))) {
682 		/* Start I/O for the buffer. */
683 		SET(bp->b_flags, B_READ | async);
684 		if (async)
685 			BIO_SETPRIO(bp, BPRIO_TIMELIMITED);
686 		else
687 			BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
688 		VOP_STRATEGY(vp, bp);
689 
690 		/* Pay for the read. */
691 		curlwp->l_ru.ru_inblock++;
692 	} else if (async)
693 		brelse(bp, 0);
694 
695 	if (vp->v_type == VBLK)
696 		mp = spec_node_getmountedfs(vp);
697 	else
698 		mp = vp->v_mount;
699 
700 	/*
701 	 * Collect statistics on synchronous and asynchronous reads.
702 	 * Reads from block devices are charged to their associated
703 	 * filesystem (if any).
704 	 */
705 	if (mp != NULL) {
706 		if (async == 0)
707 			mp->mnt_stat.f_syncreads++;
708 		else
709 			mp->mnt_stat.f_asyncreads++;
710 	}
711 
712 	return (bp);
713 }
714 
715 /*
716  * Read a disk block.
717  * This algorithm described in Bach (p.54).
718  */
719 int
720 bread(struct vnode *vp, daddr_t blkno, int size, int flags, buf_t **bpp)
721 {
722 	buf_t *bp;
723 	int error;
724 
725 	/* Get buffer for block. */
726 	bp = *bpp = bio_doread(vp, blkno, size, 0);
727 	if (bp == NULL)
728 		return ENOMEM;
729 
730 	/* Wait for the read to complete, and return result. */
731 	error = biowait(bp);
732 	if (error == 0 && (flags & B_MODIFY) != 0)
733 		error = fscow_run(bp, true);
734 	if (error) {
735 		brelse(bp, 0);
736 		*bpp = NULL;
737 	}
738 
739 	return error;
740 }
741 
742 /*
743  * Read-ahead multiple disk blocks. The first is sync, the rest async.
744  * Trivial modification to the breada algorithm presented in Bach (p.55).
745  */
746 int
747 breadn(struct vnode *vp, daddr_t blkno, int size, daddr_t *rablks,
748     int *rasizes, int nrablks, int flags, buf_t **bpp)
749 {
750 	buf_t *bp;
751 	int error, i;
752 
753 	bp = *bpp = bio_doread(vp, blkno, size, 0);
754 	if (bp == NULL)
755 		return ENOMEM;
756 
757 	/*
758 	 * For each of the read-ahead blocks, start a read, if necessary.
759 	 */
760 	mutex_enter(&bufcache_lock);
761 	for (i = 0; i < nrablks; i++) {
762 		/* If it's in the cache, just go on to next one. */
763 		if (incore(vp, rablks[i]))
764 			continue;
765 
766 		/* Get a buffer for the read-ahead block */
767 		mutex_exit(&bufcache_lock);
768 		(void) bio_doread(vp, rablks[i], rasizes[i], B_ASYNC);
769 		mutex_enter(&bufcache_lock);
770 	}
771 	mutex_exit(&bufcache_lock);
772 
773 	/* Otherwise, we had to start a read for it; wait until it's valid. */
774 	error = biowait(bp);
775 	if (error == 0 && (flags & B_MODIFY) != 0)
776 		error = fscow_run(bp, true);
777 	if (error) {
778 		brelse(bp, 0);
779 		*bpp = NULL;
780 	}
781 
782 	return error;
783 }
784 
785 /*
786  * Block write.  Described in Bach (p.56)
787  */
788 int
789 bwrite(buf_t *bp)
790 {
791 	int rv, sync, wasdelayed;
792 	struct vnode *vp;
793 	struct mount *mp;
794 
795 	KASSERT(ISSET(bp->b_cflags, BC_BUSY));
796 	KASSERT(!cv_has_waiters(&bp->b_done));
797 
798 	vp = bp->b_vp;
799 	if (vp != NULL) {
800 		KASSERT(bp->b_objlock == vp->v_interlock);
801 		if (vp->v_type == VBLK)
802 			mp = spec_node_getmountedfs(vp);
803 		else
804 			mp = vp->v_mount;
805 	} else {
806 		mp = NULL;
807 	}
808 
809 	if (mp && mp->mnt_wapbl) {
810 		if (bp->b_iodone != mp->mnt_wapbl_op->wo_wapbl_biodone) {
811 			bdwrite(bp);
812 			return 0;
813 		}
814 	}
815 
816 	/*
817 	 * Remember buffer type, to switch on it later.  If the write was
818 	 * synchronous, but the file system was mounted with MNT_ASYNC,
819 	 * convert it to a delayed write.
820 	 * XXX note that this relies on delayed tape writes being converted
821 	 * to async, not sync writes (which is safe, but ugly).
822 	 */
823 	sync = !ISSET(bp->b_flags, B_ASYNC);
824 	if (sync && mp != NULL && ISSET(mp->mnt_flag, MNT_ASYNC)) {
825 		bdwrite(bp);
826 		return (0);
827 	}
828 
829 	/*
830 	 * Collect statistics on synchronous and asynchronous writes.
831 	 * Writes to block devices are charged to their associated
832 	 * filesystem (if any).
833 	 */
834 	if (mp != NULL) {
835 		if (sync)
836 			mp->mnt_stat.f_syncwrites++;
837 		else
838 			mp->mnt_stat.f_asyncwrites++;
839 	}
840 
841 	/*
842 	 * Pay for the I/O operation and make sure the buf is on the correct
843 	 * vnode queue.
844 	 */
845 	bp->b_error = 0;
846 	wasdelayed = ISSET(bp->b_oflags, BO_DELWRI);
847 	CLR(bp->b_flags, B_READ);
848 	if (wasdelayed) {
849 		mutex_enter(&bufcache_lock);
850 		mutex_enter(bp->b_objlock);
851 		CLR(bp->b_oflags, BO_DONE | BO_DELWRI);
852 		reassignbuf(bp, bp->b_vp);
853 		mutex_exit(&bufcache_lock);
854 	} else {
855 		curlwp->l_ru.ru_oublock++;
856 		mutex_enter(bp->b_objlock);
857 		CLR(bp->b_oflags, BO_DONE | BO_DELWRI);
858 	}
859 	if (vp != NULL)
860 		vp->v_numoutput++;
861 	mutex_exit(bp->b_objlock);
862 
863 	/* Initiate disk write. */
864 	if (sync)
865 		BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
866 	else
867 		BIO_SETPRIO(bp, BPRIO_TIMELIMITED);
868 
869 	VOP_STRATEGY(vp, bp);
870 
871 	if (sync) {
872 		/* If I/O was synchronous, wait for it to complete. */
873 		rv = biowait(bp);
874 
875 		/* Release the buffer. */
876 		brelse(bp, 0);
877 
878 		return (rv);
879 	} else {
880 		return (0);
881 	}
882 }
883 
884 int
885 vn_bwrite(void *v)
886 {
887 	struct vop_bwrite_args *ap = v;
888 
889 	return (bwrite(ap->a_bp));
890 }
891 
892 /*
893  * Delayed write.
894  *
895  * The buffer is marked dirty, but is not queued for I/O.
896  * This routine should be used when the buffer is expected
897  * to be modified again soon, typically a small write that
898  * partially fills a buffer.
899  *
900  * NB: magnetic tapes cannot be delayed; they must be
901  * written in the order that the writes are requested.
902  *
903  * Described in Leffler, et al. (pp. 208-213).
904  */
905 void
906 bdwrite(buf_t *bp)
907 {
908 
909 	KASSERT(bp->b_vp == NULL || bp->b_vp->v_tag != VT_UFS ||
910 	    bp->b_vp->v_type == VBLK || ISSET(bp->b_flags, B_COWDONE));
911 	KASSERT(ISSET(bp->b_cflags, BC_BUSY));
912 	KASSERT(!cv_has_waiters(&bp->b_done));
913 
914 	/* If this is a tape block, write the block now. */
915 	if (bdev_type(bp->b_dev) == D_TAPE) {
916 		bawrite(bp);
917 		return;
918 	}
919 
920 	if (wapbl_vphaswapbl(bp->b_vp)) {
921 		struct mount *mp = wapbl_vptomp(bp->b_vp);
922 
923 		if (bp->b_iodone != mp->mnt_wapbl_op->wo_wapbl_biodone) {
924 			WAPBL_ADD_BUF(mp, bp);
925 		}
926 	}
927 
928 	/*
929 	 * If the block hasn't been seen before:
930 	 *	(1) Mark it as having been seen,
931 	 *	(2) Charge for the write,
932 	 *	(3) Make sure it's on its vnode's correct block list.
933 	 */
934 	KASSERT(bp->b_vp == NULL || bp->b_objlock == bp->b_vp->v_interlock);
935 
936 	if (!ISSET(bp->b_oflags, BO_DELWRI)) {
937 		mutex_enter(&bufcache_lock);
938 		mutex_enter(bp->b_objlock);
939 		SET(bp->b_oflags, BO_DELWRI);
940 		curlwp->l_ru.ru_oublock++;
941 		reassignbuf(bp, bp->b_vp);
942 		mutex_exit(&bufcache_lock);
943 	} else {
944 		mutex_enter(bp->b_objlock);
945 	}
946 	/* Otherwise, the "write" is done, so mark and release the buffer. */
947 	CLR(bp->b_oflags, BO_DONE);
948 	mutex_exit(bp->b_objlock);
949 
950 	brelse(bp, 0);
951 }
952 
953 /*
954  * Asynchronous block write; just an asynchronous bwrite().
955  */
956 void
957 bawrite(buf_t *bp)
958 {
959 
960 	KASSERT(ISSET(bp->b_cflags, BC_BUSY));
961 	KASSERT(bp->b_vp != NULL);
962 
963 	SET(bp->b_flags, B_ASYNC);
964 	VOP_BWRITE(bp->b_vp, bp);
965 }
966 
967 /*
968  * Release a buffer on to the free lists.
969  * Described in Bach (p. 46).
970  */
971 void
972 brelsel(buf_t *bp, int set)
973 {
974 	struct bqueue *bufq;
975 	struct vnode *vp;
976 
977 	KASSERT(bp != NULL);
978 	KASSERT(mutex_owned(&bufcache_lock));
979 	KASSERT(!cv_has_waiters(&bp->b_done));
980 	KASSERT(bp->b_refcnt > 0);
981 
982 	SET(bp->b_cflags, set);
983 
984 	KASSERT(ISSET(bp->b_cflags, BC_BUSY));
985 	KASSERT(bp->b_iodone == NULL);
986 
987 	/* Wake up any processes waiting for any buffer to become free. */
988 	cv_signal(&needbuffer_cv);
989 
990 	/* Wake up any proceeses waiting for _this_ buffer to become */
991 	if (ISSET(bp->b_cflags, BC_WANTED))
992 		CLR(bp->b_cflags, BC_WANTED|BC_AGE);
993 
994 	/* If it's clean clear the copy-on-write flag. */
995 	if (ISSET(bp->b_flags, B_COWDONE)) {
996 		mutex_enter(bp->b_objlock);
997 		if (!ISSET(bp->b_oflags, BO_DELWRI))
998 			CLR(bp->b_flags, B_COWDONE);
999 		mutex_exit(bp->b_objlock);
1000 	}
1001 
1002 	/*
1003 	 * Determine which queue the buffer should be on, then put it there.
1004 	 */
1005 
1006 	/* If it's locked, don't report an error; try again later. */
1007 	if (ISSET(bp->b_flags, B_LOCKED))
1008 		bp->b_error = 0;
1009 
1010 	/* If it's not cacheable, or an error, mark it invalid. */
1011 	if (ISSET(bp->b_cflags, BC_NOCACHE) || bp->b_error != 0)
1012 		SET(bp->b_cflags, BC_INVAL);
1013 
1014 	if (ISSET(bp->b_cflags, BC_VFLUSH)) {
1015 		/*
1016 		 * This is a delayed write buffer that was just flushed to
1017 		 * disk.  It is still on the LRU queue.  If it's become
1018 		 * invalid, then we need to move it to a different queue;
1019 		 * otherwise leave it in its current position.
1020 		 */
1021 		CLR(bp->b_cflags, BC_VFLUSH);
1022 		if (!ISSET(bp->b_cflags, BC_INVAL|BC_AGE) &&
1023 		    !ISSET(bp->b_flags, B_LOCKED) && bp->b_error == 0) {
1024 			KDASSERT(checkfreelist(bp, &bufqueues[BQ_LRU], 1));
1025 			goto already_queued;
1026 		} else {
1027 			bremfree(bp);
1028 		}
1029 	}
1030 
1031 	KDASSERT(checkfreelist(bp, &bufqueues[BQ_AGE], 0));
1032 	KDASSERT(checkfreelist(bp, &bufqueues[BQ_LRU], 0));
1033 	KDASSERT(checkfreelist(bp, &bufqueues[BQ_LOCKED], 0));
1034 
1035 	if ((bp->b_bufsize <= 0) || ISSET(bp->b_cflags, BC_INVAL)) {
1036 		/*
1037 		 * If it's invalid or empty, dissociate it from its vnode
1038 		 * and put on the head of the appropriate queue.
1039 		 */
1040 		if (ISSET(bp->b_flags, B_LOCKED)) {
1041 			if (wapbl_vphaswapbl(vp = bp->b_vp)) {
1042 				struct mount *mp = wapbl_vptomp(vp);
1043 
1044 				KASSERT(bp->b_iodone
1045 				    != mp->mnt_wapbl_op->wo_wapbl_biodone);
1046 				WAPBL_REMOVE_BUF(mp, bp);
1047 			}
1048 		}
1049 
1050 		mutex_enter(bp->b_objlock);
1051 		CLR(bp->b_oflags, BO_DONE|BO_DELWRI);
1052 		if ((vp = bp->b_vp) != NULL) {
1053 			KASSERT(bp->b_objlock == vp->v_interlock);
1054 			reassignbuf(bp, bp->b_vp);
1055 			brelvp(bp);
1056 			mutex_exit(vp->v_interlock);
1057 		} else {
1058 			KASSERT(bp->b_objlock == &buffer_lock);
1059 			mutex_exit(bp->b_objlock);
1060 		}
1061 
1062 		if (bp->b_bufsize <= 0)
1063 			/* no data */
1064 			goto already_queued;
1065 		else
1066 			/* invalid data */
1067 			bufq = &bufqueues[BQ_AGE];
1068 		binsheadfree(bp, bufq);
1069 	} else  {
1070 		/*
1071 		 * It has valid data.  Put it on the end of the appropriate
1072 		 * queue, so that it'll stick around for as long as possible.
1073 		 * If buf is AGE, but has dependencies, must put it on last
1074 		 * bufqueue to be scanned, ie LRU. This protects against the
1075 		 * livelock where BQ_AGE only has buffers with dependencies,
1076 		 * and we thus never get to the dependent buffers in BQ_LRU.
1077 		 */
1078 		if (ISSET(bp->b_flags, B_LOCKED)) {
1079 			/* locked in core */
1080 			bufq = &bufqueues[BQ_LOCKED];
1081 		} else if (!ISSET(bp->b_cflags, BC_AGE)) {
1082 			/* valid data */
1083 			bufq = &bufqueues[BQ_LRU];
1084 		} else {
1085 			/* stale but valid data */
1086 			bufq = &bufqueues[BQ_AGE];
1087 		}
1088 		binstailfree(bp, bufq);
1089 	}
1090 already_queued:
1091 	/* Unlock the buffer. */
1092 	CLR(bp->b_cflags, BC_AGE|BC_BUSY|BC_NOCACHE);
1093 	CLR(bp->b_flags, B_ASYNC);
1094 	cv_broadcast(&bp->b_busy);
1095 
1096 	if (bp->b_bufsize <= 0)
1097 		brele(bp);
1098 }
1099 
1100 void
1101 brelse(buf_t *bp, int set)
1102 {
1103 
1104 	mutex_enter(&bufcache_lock);
1105 	brelsel(bp, set);
1106 	mutex_exit(&bufcache_lock);
1107 }
1108 
1109 /*
1110  * Determine if a block is in the cache.
1111  * Just look on what would be its hash chain.  If it's there, return
1112  * a pointer to it, unless it's marked invalid.  If it's marked invalid,
1113  * we normally don't return the buffer, unless the caller explicitly
1114  * wants us to.
1115  */
1116 buf_t *
1117 incore(struct vnode *vp, daddr_t blkno)
1118 {
1119 	buf_t *bp;
1120 
1121 	KASSERT(mutex_owned(&bufcache_lock));
1122 
1123 	/* Search hash chain */
1124 	LIST_FOREACH(bp, BUFHASH(vp, blkno), b_hash) {
1125 		if (bp->b_lblkno == blkno && bp->b_vp == vp &&
1126 		    !ISSET(bp->b_cflags, BC_INVAL)) {
1127 		    	KASSERT(bp->b_objlock == vp->v_interlock);
1128 		    	return (bp);
1129 		}
1130 	}
1131 
1132 	return (NULL);
1133 }
1134 
1135 /*
1136  * Get a block of requested size that is associated with
1137  * a given vnode and block offset. If it is found in the
1138  * block cache, mark it as having been found, make it busy
1139  * and return it. Otherwise, return an empty block of the
1140  * correct size. It is up to the caller to insure that the
1141  * cached blocks be of the correct size.
1142  */
1143 buf_t *
1144 getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, int slptimeo)
1145 {
1146 	int err, preserve;
1147 	buf_t *bp;
1148 
1149 	mutex_enter(&bufcache_lock);
1150  loop:
1151 	bp = incore(vp, blkno);
1152 	if (bp != NULL) {
1153 		err = bbusy(bp, ((slpflag & PCATCH) != 0), slptimeo, NULL);
1154 		if (err != 0) {
1155 			if (err == EPASSTHROUGH)
1156 				goto loop;
1157 			mutex_exit(&bufcache_lock);
1158 			return (NULL);
1159 		}
1160 		KASSERT(!cv_has_waiters(&bp->b_done));
1161 #ifdef DIAGNOSTIC
1162 		if (ISSET(bp->b_oflags, BO_DONE|BO_DELWRI) &&
1163 		    bp->b_bcount < size && vp->v_type != VBLK)
1164 			panic("getblk: block size invariant failed");
1165 #endif
1166 		bremfree(bp);
1167 		preserve = 1;
1168 	} else {
1169 		if ((bp = getnewbuf(slpflag, slptimeo, 0)) == NULL)
1170 			goto loop;
1171 
1172 		if (incore(vp, blkno) != NULL) {
1173 			/* The block has come into memory in the meantime. */
1174 			brelsel(bp, 0);
1175 			goto loop;
1176 		}
1177 
1178 		LIST_INSERT_HEAD(BUFHASH(vp, blkno), bp, b_hash);
1179 		bp->b_blkno = bp->b_lblkno = bp->b_rawblkno = blkno;
1180 		mutex_enter(vp->v_interlock);
1181 		bgetvp(vp, bp);
1182 		mutex_exit(vp->v_interlock);
1183 		preserve = 0;
1184 	}
1185 	mutex_exit(&bufcache_lock);
1186 
1187 	/*
1188 	 * LFS can't track total size of B_LOCKED buffer (locked_queue_bytes)
1189 	 * if we re-size buffers here.
1190 	 */
1191 	if (ISSET(bp->b_flags, B_LOCKED)) {
1192 		KASSERT(bp->b_bufsize >= size);
1193 	} else {
1194 		if (allocbuf(bp, size, preserve)) {
1195 			mutex_enter(&bufcache_lock);
1196 			LIST_REMOVE(bp, b_hash);
1197 			mutex_exit(&bufcache_lock);
1198 			brelse(bp, BC_INVAL);
1199 			return NULL;
1200 		}
1201 	}
1202 	BIO_SETPRIO(bp, BPRIO_DEFAULT);
1203 	return (bp);
1204 }
1205 
1206 /*
1207  * Get an empty, disassociated buffer of given size.
1208  */
1209 buf_t *
1210 geteblk(int size)
1211 {
1212 	buf_t *bp;
1213 	int error __diagused;
1214 
1215 	mutex_enter(&bufcache_lock);
1216 	while ((bp = getnewbuf(0, 0, 0)) == NULL)
1217 		;
1218 
1219 	SET(bp->b_cflags, BC_INVAL);
1220 	LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1221 	mutex_exit(&bufcache_lock);
1222 	BIO_SETPRIO(bp, BPRIO_DEFAULT);
1223 	error = allocbuf(bp, size, 0);
1224 	KASSERT(error == 0);
1225 	return (bp);
1226 }
1227 
1228 /*
1229  * Expand or contract the actual memory allocated to a buffer.
1230  *
1231  * If the buffer shrinks, data is lost, so it's up to the
1232  * caller to have written it out *first*; this routine will not
1233  * start a write.  If the buffer grows, it's the callers
1234  * responsibility to fill out the buffer's additional contents.
1235  */
1236 int
1237 allocbuf(buf_t *bp, int size, int preserve)
1238 {
1239 	void *addr;
1240 	vsize_t oldsize, desired_size;
1241 	int oldcount;
1242 	int delta;
1243 
1244 	desired_size = buf_roundsize(size);
1245 	if (desired_size > MAXBSIZE)
1246 		printf("allocbuf: buffer larger than MAXBSIZE requested");
1247 
1248 	oldcount = bp->b_bcount;
1249 
1250 	bp->b_bcount = size;
1251 
1252 	oldsize = bp->b_bufsize;
1253 	if (oldsize == desired_size) {
1254 		/*
1255 		 * Do not short cut the WAPBL resize, as the buffer length
1256 		 * could still have changed and this would corrupt the
1257 		 * tracking of the transaction length.
1258 		 */
1259 		goto out;
1260 	}
1261 
1262 	/*
1263 	 * If we want a buffer of a different size, re-allocate the
1264 	 * buffer's memory; copy old content only if needed.
1265 	 */
1266 	addr = buf_alloc(desired_size);
1267 	if (addr == NULL)
1268 		return ENOMEM;
1269 	if (preserve)
1270 		memcpy(addr, bp->b_data, MIN(oldsize,desired_size));
1271 	if (bp->b_data != NULL)
1272 		buf_mrelease(bp->b_data, oldsize);
1273 	bp->b_data = addr;
1274 	bp->b_bufsize = desired_size;
1275 
1276 	/*
1277 	 * Update overall buffer memory counter (protected by bufcache_lock)
1278 	 */
1279 	delta = (long)desired_size - (long)oldsize;
1280 
1281 	mutex_enter(&bufcache_lock);
1282 	if ((bufmem += delta) > bufmem_hiwater) {
1283 		/*
1284 		 * Need to trim overall memory usage.
1285 		 */
1286 		while (buf_canrelease()) {
1287 			if (curcpu()->ci_schedstate.spc_flags &
1288 			    SPCF_SHOULDYIELD) {
1289 				mutex_exit(&bufcache_lock);
1290 				preempt();
1291 				mutex_enter(&bufcache_lock);
1292 			}
1293 			if (buf_trim() == 0)
1294 				break;
1295 		}
1296 	}
1297 	mutex_exit(&bufcache_lock);
1298 
1299  out:
1300 	if (wapbl_vphaswapbl(bp->b_vp))
1301 		WAPBL_RESIZE_BUF(wapbl_vptomp(bp->b_vp), bp, oldsize, oldcount);
1302 
1303 	return 0;
1304 }
1305 
1306 /*
1307  * Find a buffer which is available for use.
1308  * Select something from a free list.
1309  * Preference is to AGE list, then LRU list.
1310  *
1311  * Called with the buffer queues locked.
1312  * Return buffer locked.
1313  */
1314 buf_t *
1315 getnewbuf(int slpflag, int slptimeo, int from_bufq)
1316 {
1317 	buf_t *bp;
1318 	struct vnode *vp;
1319 
1320  start:
1321 	KASSERT(mutex_owned(&bufcache_lock));
1322 
1323 	/*
1324 	 * Get a new buffer from the pool.
1325 	 */
1326 	if (!from_bufq && buf_lotsfree()) {
1327 		mutex_exit(&bufcache_lock);
1328 		bp = pool_cache_get(buf_cache, PR_NOWAIT);
1329 		if (bp != NULL) {
1330 			memset((char *)bp, 0, sizeof(*bp));
1331 			buf_init(bp);
1332 			SET(bp->b_cflags, BC_BUSY);	/* mark buffer busy */
1333 			mutex_enter(&bufcache_lock);
1334 #if defined(DIAGNOSTIC)
1335 			bp->b_freelistindex = -1;
1336 #endif /* defined(DIAGNOSTIC) */
1337 			return (bp);
1338 		}
1339 		mutex_enter(&bufcache_lock);
1340 	}
1341 
1342 	KASSERT(mutex_owned(&bufcache_lock));
1343 	if ((bp = TAILQ_FIRST(&bufqueues[BQ_AGE].bq_queue)) != NULL ||
1344 	    (bp = TAILQ_FIRST(&bufqueues[BQ_LRU].bq_queue)) != NULL) {
1345 	    	KASSERT(!ISSET(bp->b_cflags, BC_BUSY) || ISSET(bp->b_cflags, BC_VFLUSH));
1346 		bremfree(bp);
1347 
1348 		/* Buffer is no longer on free lists. */
1349 		SET(bp->b_cflags, BC_BUSY);
1350 	} else {
1351 		/*
1352 		 * XXX: !from_bufq should be removed.
1353 		 */
1354 		if (!from_bufq || curlwp != uvm.pagedaemon_lwp) {
1355 			/* wait for a free buffer of any kind */
1356 			if ((slpflag & PCATCH) != 0)
1357 				(void)cv_timedwait_sig(&needbuffer_cv,
1358 				    &bufcache_lock, slptimeo);
1359 			else
1360 				(void)cv_timedwait(&needbuffer_cv,
1361 				    &bufcache_lock, slptimeo);
1362 		}
1363 		return (NULL);
1364 	}
1365 
1366 #ifdef DIAGNOSTIC
1367 	if (bp->b_bufsize <= 0)
1368 		panic("buffer %p: on queue but empty", bp);
1369 #endif
1370 
1371 	if (ISSET(bp->b_cflags, BC_VFLUSH)) {
1372 		/*
1373 		 * This is a delayed write buffer being flushed to disk.  Make
1374 		 * sure it gets aged out of the queue when it's finished, and
1375 		 * leave it off the LRU queue.
1376 		 */
1377 		CLR(bp->b_cflags, BC_VFLUSH);
1378 		SET(bp->b_cflags, BC_AGE);
1379 		goto start;
1380 	}
1381 
1382 	KASSERT(ISSET(bp->b_cflags, BC_BUSY));
1383 	KASSERT(bp->b_refcnt > 0);
1384     	KASSERT(!cv_has_waiters(&bp->b_done));
1385 
1386 	/*
1387 	 * If buffer was a delayed write, start it and return NULL
1388 	 * (since we might sleep while starting the write).
1389 	 */
1390 	if (ISSET(bp->b_oflags, BO_DELWRI)) {
1391 		/*
1392 		 * This buffer has gone through the LRU, so make sure it gets
1393 		 * reused ASAP.
1394 		 */
1395 		SET(bp->b_cflags, BC_AGE);
1396 		mutex_exit(&bufcache_lock);
1397 		bawrite(bp);
1398 		mutex_enter(&bufcache_lock);
1399 		return (NULL);
1400 	}
1401 
1402 	vp = bp->b_vp;
1403 
1404 	/* clear out various other fields */
1405 	bp->b_cflags = BC_BUSY;
1406 	bp->b_oflags = 0;
1407 	bp->b_flags = 0;
1408 	bp->b_dev = NODEV;
1409 	bp->b_blkno = 0;
1410 	bp->b_lblkno = 0;
1411 	bp->b_rawblkno = 0;
1412 	bp->b_iodone = 0;
1413 	bp->b_error = 0;
1414 	bp->b_resid = 0;
1415 	bp->b_bcount = 0;
1416 
1417 	LIST_REMOVE(bp, b_hash);
1418 
1419 	/* Disassociate us from our vnode, if we had one... */
1420 	if (vp != NULL) {
1421 		mutex_enter(vp->v_interlock);
1422 		brelvp(bp);
1423 		mutex_exit(vp->v_interlock);
1424 	}
1425 
1426 	return (bp);
1427 }
1428 
1429 /*
1430  * Attempt to free an aged buffer off the queues.
1431  * Called with queue lock held.
1432  * Returns the amount of buffer memory freed.
1433  */
1434 static int
1435 buf_trim(void)
1436 {
1437 	buf_t *bp;
1438 	long size;
1439 
1440 	KASSERT(mutex_owned(&bufcache_lock));
1441 
1442 	/* Instruct getnewbuf() to get buffers off the queues */
1443 	if ((bp = getnewbuf(PCATCH, 1, 1)) == NULL)
1444 		return 0;
1445 
1446 	KASSERT((bp->b_cflags & BC_WANTED) == 0);
1447 	size = bp->b_bufsize;
1448 	bufmem -= size;
1449 	if (size > 0) {
1450 		buf_mrelease(bp->b_data, size);
1451 		bp->b_bcount = bp->b_bufsize = 0;
1452 	}
1453 	/* brelse() will return the buffer to the global buffer pool */
1454 	brelsel(bp, 0);
1455 	return size;
1456 }
1457 
1458 int
1459 buf_drain(int n)
1460 {
1461 	int size = 0, sz;
1462 
1463 	KASSERT(mutex_owned(&bufcache_lock));
1464 
1465 	while (size < n && bufmem > bufmem_lowater) {
1466 		sz = buf_trim();
1467 		if (sz <= 0)
1468 			break;
1469 		size += sz;
1470 	}
1471 
1472 	return size;
1473 }
1474 
1475 /*
1476  * Wait for operations on the buffer to complete.
1477  * When they do, extract and return the I/O's error value.
1478  */
1479 int
1480 biowait(buf_t *bp)
1481 {
1482 
1483 	KASSERT(ISSET(bp->b_cflags, BC_BUSY));
1484 	KASSERT(bp->b_refcnt > 0);
1485 
1486 	mutex_enter(bp->b_objlock);
1487 	while (!ISSET(bp->b_oflags, BO_DONE | BO_DELWRI))
1488 		cv_wait(&bp->b_done, bp->b_objlock);
1489 	mutex_exit(bp->b_objlock);
1490 
1491 	return bp->b_error;
1492 }
1493 
1494 /*
1495  * Mark I/O complete on a buffer.
1496  *
1497  * If a callback has been requested, e.g. the pageout
1498  * daemon, do so. Otherwise, awaken waiting processes.
1499  *
1500  * [ Leffler, et al., says on p.247:
1501  *	"This routine wakes up the blocked process, frees the buffer
1502  *	for an asynchronous write, or, for a request by the pagedaemon
1503  *	process, invokes a procedure specified in the buffer structure" ]
1504  *
1505  * In real life, the pagedaemon (or other system processes) wants
1506  * to do async stuff to, and doesn't want the buffer brelse()'d.
1507  * (for swap pager, that puts swap buffers on the free lists (!!!),
1508  * for the vn device, that puts allocated buffers on the free lists!)
1509  */
1510 void
1511 biodone(buf_t *bp)
1512 {
1513 	int s;
1514 
1515 	KASSERT(!ISSET(bp->b_oflags, BO_DONE));
1516 
1517 	if (cpu_intr_p()) {
1518 		/* From interrupt mode: defer to a soft interrupt. */
1519 		s = splvm();
1520 		TAILQ_INSERT_TAIL(&curcpu()->ci_data.cpu_biodone, bp, b_actq);
1521 		softint_schedule(biodone_sih);
1522 		splx(s);
1523 	} else {
1524 		/* Process now - the buffer may be freed soon. */
1525 		biodone2(bp);
1526 	}
1527 }
1528 
1529 static void
1530 biodone2(buf_t *bp)
1531 {
1532 	void (*callout)(buf_t *);
1533 
1534 	mutex_enter(bp->b_objlock);
1535 	/* Note that the transfer is done. */
1536 	if (ISSET(bp->b_oflags, BO_DONE))
1537 		panic("biodone2 already");
1538 	CLR(bp->b_flags, B_COWDONE);
1539 	SET(bp->b_oflags, BO_DONE);
1540 	BIO_SETPRIO(bp, BPRIO_DEFAULT);
1541 
1542 	/* Wake up waiting writers. */
1543 	if (!ISSET(bp->b_flags, B_READ))
1544 		vwakeup(bp);
1545 
1546 	if ((callout = bp->b_iodone) != NULL) {
1547 		/* Note callout done, then call out. */
1548 		KASSERT(!cv_has_waiters(&bp->b_done));
1549 		KERNEL_LOCK(1, NULL);		/* XXXSMP */
1550 		bp->b_iodone = NULL;
1551 		mutex_exit(bp->b_objlock);
1552 		(*callout)(bp);
1553 		KERNEL_UNLOCK_ONE(NULL);	/* XXXSMP */
1554 	} else if (ISSET(bp->b_flags, B_ASYNC)) {
1555 		/* If async, release. */
1556 		KASSERT(!cv_has_waiters(&bp->b_done));
1557 		mutex_exit(bp->b_objlock);
1558 		brelse(bp, 0);
1559 	} else {
1560 		/* Otherwise just wake up waiters in biowait(). */
1561 		cv_broadcast(&bp->b_done);
1562 		mutex_exit(bp->b_objlock);
1563 	}
1564 }
1565 
1566 static void
1567 biointr(void *cookie)
1568 {
1569 	struct cpu_info *ci;
1570 	buf_t *bp;
1571 	int s;
1572 
1573 	ci = curcpu();
1574 
1575 	while (!TAILQ_EMPTY(&ci->ci_data.cpu_biodone)) {
1576 		KASSERT(curcpu() == ci);
1577 
1578 		s = splvm();
1579 		bp = TAILQ_FIRST(&ci->ci_data.cpu_biodone);
1580 		TAILQ_REMOVE(&ci->ci_data.cpu_biodone, bp, b_actq);
1581 		splx(s);
1582 
1583 		biodone2(bp);
1584 	}
1585 }
1586 
1587 /*
1588  * Wait for all buffers to complete I/O
1589  * Return the number of "stuck" buffers.
1590  */
1591 int
1592 buf_syncwait(void)
1593 {
1594 	buf_t *bp;
1595 	int iter, nbusy, nbusy_prev = 0, ihash;
1596 
1597 	for (iter = 0; iter < 20;) {
1598 		mutex_enter(&bufcache_lock);
1599 		nbusy = 0;
1600 		for (ihash = 0; ihash < bufhash+1; ihash++) {
1601 		    LIST_FOREACH(bp, &bufhashtbl[ihash], b_hash) {
1602 			if ((bp->b_cflags & (BC_BUSY|BC_INVAL)) == BC_BUSY)
1603 				nbusy += ((bp->b_flags & B_READ) == 0);
1604 		    }
1605 		}
1606 		mutex_exit(&bufcache_lock);
1607 
1608 		if (nbusy == 0)
1609 			break;
1610 		if (nbusy_prev == 0)
1611 			nbusy_prev = nbusy;
1612 		printf("%d ", nbusy);
1613 		kpause("bflush", false, MAX(1, hz / 25 * iter), NULL);
1614 		if (nbusy >= nbusy_prev) /* we didn't flush anything */
1615 			iter++;
1616 		else
1617 			nbusy_prev = nbusy;
1618 	}
1619 
1620 	if (nbusy) {
1621 #if defined(DEBUG) || defined(DEBUG_HALT_BUSY)
1622 		printf("giving up\nPrinting vnodes for busy buffers\n");
1623 		for (ihash = 0; ihash < bufhash+1; ihash++) {
1624 		    LIST_FOREACH(bp, &bufhashtbl[ihash], b_hash) {
1625 			if ((bp->b_cflags & (BC_BUSY|BC_INVAL)) == BC_BUSY &&
1626 			    (bp->b_flags & B_READ) == 0)
1627 				vprint(NULL, bp->b_vp);
1628 		    }
1629 		}
1630 #endif
1631 	}
1632 
1633 	return nbusy;
1634 }
1635 
1636 static void
1637 sysctl_fillbuf(buf_t *i, struct buf_sysctl *o)
1638 {
1639 
1640 	o->b_flags = i->b_flags | i->b_cflags | i->b_oflags;
1641 	o->b_error = i->b_error;
1642 	o->b_prio = i->b_prio;
1643 	o->b_dev = i->b_dev;
1644 	o->b_bufsize = i->b_bufsize;
1645 	o->b_bcount = i->b_bcount;
1646 	o->b_resid = i->b_resid;
1647 	o->b_addr = PTRTOUINT64(i->b_data);
1648 	o->b_blkno = i->b_blkno;
1649 	o->b_rawblkno = i->b_rawblkno;
1650 	o->b_iodone = PTRTOUINT64(i->b_iodone);
1651 	o->b_proc = PTRTOUINT64(i->b_proc);
1652 	o->b_vp = PTRTOUINT64(i->b_vp);
1653 	o->b_saveaddr = PTRTOUINT64(i->b_saveaddr);
1654 	o->b_lblkno = i->b_lblkno;
1655 }
1656 
1657 #define KERN_BUFSLOP 20
1658 static int
1659 sysctl_dobuf(SYSCTLFN_ARGS)
1660 {
1661 	buf_t *bp;
1662 	struct buf_sysctl bs;
1663 	struct bqueue *bq;
1664 	char *dp;
1665 	u_int i, op, arg;
1666 	size_t len, needed, elem_size, out_size;
1667 	int error, elem_count, retries;
1668 
1669 	if (namelen == 1 && name[0] == CTL_QUERY)
1670 		return (sysctl_query(SYSCTLFN_CALL(rnode)));
1671 
1672 	if (namelen != 4)
1673 		return (EINVAL);
1674 
1675 	retries = 100;
1676  retry:
1677 	dp = oldp;
1678 	len = (oldp != NULL) ? *oldlenp : 0;
1679 	op = name[0];
1680 	arg = name[1];
1681 	elem_size = name[2];
1682 	elem_count = name[3];
1683 	out_size = MIN(sizeof(bs), elem_size);
1684 
1685 	/*
1686 	 * at the moment, these are just "placeholders" to make the
1687 	 * API for retrieving kern.buf data more extensible in the
1688 	 * future.
1689 	 *
1690 	 * XXX kern.buf currently has "netbsd32" issues.  hopefully
1691 	 * these will be resolved at a later point.
1692 	 */
1693 	if (op != KERN_BUF_ALL || arg != KERN_BUF_ALL ||
1694 	    elem_size < 1 || elem_count < 0)
1695 		return (EINVAL);
1696 
1697 	error = 0;
1698 	needed = 0;
1699 	sysctl_unlock();
1700 	mutex_enter(&bufcache_lock);
1701 	for (i = 0; i < BQUEUES; i++) {
1702 		bq = &bufqueues[i];
1703 		TAILQ_FOREACH(bp, &bq->bq_queue, b_freelist) {
1704 			bq->bq_marker = bp;
1705 			if (len >= elem_size && elem_count > 0) {
1706 				sysctl_fillbuf(bp, &bs);
1707 				mutex_exit(&bufcache_lock);
1708 				error = copyout(&bs, dp, out_size);
1709 				mutex_enter(&bufcache_lock);
1710 				if (error)
1711 					break;
1712 				if (bq->bq_marker != bp) {
1713 					/*
1714 					 * This sysctl node is only for
1715 					 * statistics.  Retry; if the
1716 					 * queue keeps changing, then
1717 					 * bail out.
1718 					 */
1719 					if (retries-- == 0) {
1720 						error = EAGAIN;
1721 						break;
1722 					}
1723 					mutex_exit(&bufcache_lock);
1724 					sysctl_relock();
1725 					goto retry;
1726 				}
1727 				dp += elem_size;
1728 				len -= elem_size;
1729 			}
1730 			needed += elem_size;
1731 			if (elem_count > 0 && elem_count != INT_MAX)
1732 				elem_count--;
1733 		}
1734 		if (error != 0)
1735 			break;
1736 	}
1737 	mutex_exit(&bufcache_lock);
1738 	sysctl_relock();
1739 
1740 	*oldlenp = needed;
1741 	if (oldp == NULL)
1742 		*oldlenp += KERN_BUFSLOP * sizeof(buf_t);
1743 
1744 	return (error);
1745 }
1746 
1747 static int
1748 sysctl_bufvm_update(SYSCTLFN_ARGS)
1749 {
1750 	int error, rv;
1751 	struct sysctlnode node;
1752 	unsigned int temp_bufcache;
1753 	unsigned long temp_water;
1754 
1755 	/* Take a copy of the supplied node and its data */
1756 	node = *rnode;
1757 	if (node.sysctl_data == &bufcache) {
1758 	    node.sysctl_data = &temp_bufcache;
1759 	    temp_bufcache = *(unsigned int *)rnode->sysctl_data;
1760 	} else {
1761 	    node.sysctl_data = &temp_water;
1762 	    temp_water = *(unsigned long *)rnode->sysctl_data;
1763 	}
1764 
1765 	/* Update the copy */
1766 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
1767 	if (error || newp == NULL)
1768 		return (error);
1769 
1770 	if (rnode->sysctl_data == &bufcache) {
1771 		if (temp_bufcache > 100)
1772 			return (EINVAL);
1773 		bufcache = temp_bufcache;
1774 		buf_setwm();
1775 	} else if (rnode->sysctl_data == &bufmem_lowater) {
1776 		if (bufmem_hiwater - temp_water < 16)
1777 			return (EINVAL);
1778 		bufmem_lowater = temp_water;
1779 	} else if (rnode->sysctl_data == &bufmem_hiwater) {
1780 		if (temp_water - bufmem_lowater < 16)
1781 			return (EINVAL);
1782 		bufmem_hiwater = temp_water;
1783 	} else
1784 		return (EINVAL);
1785 
1786 	/* Drain until below new high water mark */
1787 	sysctl_unlock();
1788 	mutex_enter(&bufcache_lock);
1789 	while (bufmem > bufmem_hiwater) {
1790 		rv = buf_drain((bufmem - bufmem_hiwater) / (2 * 1024));
1791 		if (rv <= 0)
1792 			break;
1793 	}
1794 	mutex_exit(&bufcache_lock);
1795 	sysctl_relock();
1796 
1797 	return 0;
1798 }
1799 
1800 static struct sysctllog *vfsbio_sysctllog;
1801 
1802 static void
1803 sysctl_kern_buf_setup(void)
1804 {
1805 
1806 	sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1807 		       CTLFLAG_PERMANENT,
1808 		       CTLTYPE_NODE, "buf",
1809 		       SYSCTL_DESCR("Kernel buffer cache information"),
1810 		       sysctl_dobuf, 0, NULL, 0,
1811 		       CTL_KERN, KERN_BUF, CTL_EOL);
1812 }
1813 
1814 static void
1815 sysctl_vm_buf_setup(void)
1816 {
1817 
1818 	sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1819 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1820 		       CTLTYPE_INT, "bufcache",
1821 		       SYSCTL_DESCR("Percentage of physical memory to use for "
1822 				    "buffer cache"),
1823 		       sysctl_bufvm_update, 0, &bufcache, 0,
1824 		       CTL_VM, CTL_CREATE, CTL_EOL);
1825 	sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1826 		       CTLFLAG_PERMANENT|CTLFLAG_READONLY,
1827 		       CTLTYPE_LONG, "bufmem",
1828 		       SYSCTL_DESCR("Amount of kernel memory used by buffer "
1829 				    "cache"),
1830 		       NULL, 0, &bufmem, 0,
1831 		       CTL_VM, CTL_CREATE, CTL_EOL);
1832 	sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1833 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1834 		       CTLTYPE_LONG, "bufmem_lowater",
1835 		       SYSCTL_DESCR("Minimum amount of kernel memory to "
1836 				    "reserve for buffer cache"),
1837 		       sysctl_bufvm_update, 0, &bufmem_lowater, 0,
1838 		       CTL_VM, CTL_CREATE, CTL_EOL);
1839 	sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1840 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1841 		       CTLTYPE_LONG, "bufmem_hiwater",
1842 		       SYSCTL_DESCR("Maximum amount of kernel memory to use "
1843 				    "for buffer cache"),
1844 		       sysctl_bufvm_update, 0, &bufmem_hiwater, 0,
1845 		       CTL_VM, CTL_CREATE, CTL_EOL);
1846 }
1847 
1848 #ifdef DEBUG
1849 /*
1850  * Print out statistics on the current allocation of the buffer pool.
1851  * Can be enabled to print out on every ``sync'' by setting "syncprt"
1852  * in vfs_syscalls.c using sysctl.
1853  */
1854 void
1855 vfs_bufstats(void)
1856 {
1857 	int i, j, count;
1858 	buf_t *bp;
1859 	struct bqueue *dp;
1860 	int counts[(MAXBSIZE / PAGE_SIZE) + 1];
1861 	static const char *bname[BQUEUES] = { "LOCKED", "LRU", "AGE" };
1862 
1863 	for (dp = bufqueues, i = 0; dp < &bufqueues[BQUEUES]; dp++, i++) {
1864 		count = 0;
1865 		for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++)
1866 			counts[j] = 0;
1867 		TAILQ_FOREACH(bp, &dp->bq_queue, b_freelist) {
1868 			counts[bp->b_bufsize/PAGE_SIZE]++;
1869 			count++;
1870 		}
1871 		printf("%s: total-%d", bname[i], count);
1872 		for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++)
1873 			if (counts[j] != 0)
1874 				printf(", %d-%d", j * PAGE_SIZE, counts[j]);
1875 		printf("\n");
1876 	}
1877 }
1878 #endif /* DEBUG */
1879 
1880 /* ------------------------------ */
1881 
1882 buf_t *
1883 getiobuf(struct vnode *vp, bool waitok)
1884 {
1885 	buf_t *bp;
1886 
1887 	bp = pool_cache_get(bufio_cache, (waitok ? PR_WAITOK : PR_NOWAIT));
1888 	if (bp == NULL)
1889 		return bp;
1890 
1891 	buf_init(bp);
1892 
1893 	if ((bp->b_vp = vp) == NULL)
1894 		bp->b_objlock = &buffer_lock;
1895 	else
1896 		bp->b_objlock = vp->v_interlock;
1897 
1898 	return bp;
1899 }
1900 
1901 void
1902 putiobuf(buf_t *bp)
1903 {
1904 
1905 	buf_destroy(bp);
1906 	pool_cache_put(bufio_cache, bp);
1907 }
1908 
1909 /*
1910  * nestiobuf_iodone: b_iodone callback for nested buffers.
1911  */
1912 
1913 void
1914 nestiobuf_iodone(buf_t *bp)
1915 {
1916 	buf_t *mbp = bp->b_private;
1917 	int error;
1918 	int donebytes;
1919 
1920 	KASSERT(bp->b_bcount <= bp->b_bufsize);
1921 	KASSERT(mbp != bp);
1922 
1923 	error = bp->b_error;
1924 	if (bp->b_error == 0 &&
1925 	    (bp->b_bcount < bp->b_bufsize || bp->b_resid > 0)) {
1926 		/*
1927 		 * Not all got transfered, raise an error. We have no way to
1928 		 * propagate these conditions to mbp.
1929 		 */
1930 		error = EIO;
1931 	}
1932 
1933 	donebytes = bp->b_bufsize;
1934 
1935 	putiobuf(bp);
1936 	nestiobuf_done(mbp, donebytes, error);
1937 }
1938 
1939 /*
1940  * nestiobuf_setup: setup a "nested" buffer.
1941  *
1942  * => 'mbp' is a "master" buffer which is being divided into sub pieces.
1943  * => 'bp' should be a buffer allocated by getiobuf.
1944  * => 'offset' is a byte offset in the master buffer.
1945  * => 'size' is a size in bytes of this nested buffer.
1946  */
1947 
1948 void
1949 nestiobuf_setup(buf_t *mbp, buf_t *bp, int offset, size_t size)
1950 {
1951 	const int b_read = mbp->b_flags & B_READ;
1952 	struct vnode *vp = mbp->b_vp;
1953 
1954 	KASSERT(mbp->b_bcount >= offset + size);
1955 	bp->b_vp = vp;
1956 	bp->b_dev = mbp->b_dev;
1957 	bp->b_objlock = mbp->b_objlock;
1958 	bp->b_cflags = BC_BUSY;
1959 	bp->b_flags = B_ASYNC | b_read;
1960 	bp->b_iodone = nestiobuf_iodone;
1961 	bp->b_data = (char *)mbp->b_data + offset;
1962 	bp->b_resid = bp->b_bcount = size;
1963 	bp->b_bufsize = bp->b_bcount;
1964 	bp->b_private = mbp;
1965 	BIO_COPYPRIO(bp, mbp);
1966 	if (!b_read && vp != NULL) {
1967 		mutex_enter(vp->v_interlock);
1968 		vp->v_numoutput++;
1969 		mutex_exit(vp->v_interlock);
1970 	}
1971 }
1972 
1973 /*
1974  * nestiobuf_done: propagate completion to the master buffer.
1975  *
1976  * => 'donebytes' specifies how many bytes in the 'mbp' is completed.
1977  * => 'error' is an errno(2) that 'donebytes' has been completed with.
1978  */
1979 
1980 void
1981 nestiobuf_done(buf_t *mbp, int donebytes, int error)
1982 {
1983 
1984 	if (donebytes == 0) {
1985 		return;
1986 	}
1987 	mutex_enter(mbp->b_objlock);
1988 	KASSERT(mbp->b_resid >= donebytes);
1989 	mbp->b_resid -= donebytes;
1990 	if (error)
1991 		mbp->b_error = error;
1992 	if (mbp->b_resid == 0) {
1993 		if (mbp->b_error)
1994 			mbp->b_resid = mbp->b_bcount;
1995 		mutex_exit(mbp->b_objlock);
1996 		biodone(mbp);
1997 	} else
1998 		mutex_exit(mbp->b_objlock);
1999 }
2000 
2001 void
2002 buf_init(buf_t *bp)
2003 {
2004 
2005 	cv_init(&bp->b_busy, "biolock");
2006 	cv_init(&bp->b_done, "biowait");
2007 	bp->b_dev = NODEV;
2008 	bp->b_error = 0;
2009 	bp->b_flags = 0;
2010 	bp->b_cflags = 0;
2011 	bp->b_oflags = 0;
2012 	bp->b_objlock = &buffer_lock;
2013 	bp->b_iodone = NULL;
2014 	bp->b_refcnt = 1;
2015 	bp->b_dev = NODEV;
2016 	bp->b_vnbufs.le_next = NOLIST;
2017 	BIO_SETPRIO(bp, BPRIO_DEFAULT);
2018 }
2019 
2020 void
2021 buf_destroy(buf_t *bp)
2022 {
2023 
2024 	cv_destroy(&bp->b_done);
2025 	cv_destroy(&bp->b_busy);
2026 }
2027 
2028 int
2029 bbusy(buf_t *bp, bool intr, int timo, kmutex_t *interlock)
2030 {
2031 	int error;
2032 
2033 	KASSERT(mutex_owned(&bufcache_lock));
2034 
2035 	if ((bp->b_cflags & BC_BUSY) != 0) {
2036 		if (curlwp == uvm.pagedaemon_lwp)
2037 			return EDEADLK;
2038 		bp->b_cflags |= BC_WANTED;
2039 		bref(bp);
2040 		if (interlock != NULL)
2041 			mutex_exit(interlock);
2042 		if (intr) {
2043 			error = cv_timedwait_sig(&bp->b_busy, &bufcache_lock,
2044 			    timo);
2045 		} else {
2046 			error = cv_timedwait(&bp->b_busy, &bufcache_lock,
2047 			    timo);
2048 		}
2049 		brele(bp);
2050 		if (interlock != NULL)
2051 			mutex_enter(interlock);
2052 		if (error != 0)
2053 			return error;
2054 		return EPASSTHROUGH;
2055 	}
2056 	bp->b_cflags |= BC_BUSY;
2057 
2058 	return 0;
2059 }
2060