xref: /netbsd-src/sys/kern/vfs_bio.c (revision 1b9578b8c2c1f848eeb16dabbfd7d1f0d9fdefbd)
1 /*	$NetBSD: vfs_bio.c,v 1.230 2011/06/12 03:35:56 rmind Exp $	*/
2 
3 /*-
4  * Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Andrew Doran, and by Wasabi Systems, Inc.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 /*-
33  * Copyright (c) 1982, 1986, 1989, 1993
34  *	The Regents of the University of California.  All rights reserved.
35  * (c) UNIX System Laboratories, Inc.
36  * All or some portions of this file are derived from material licensed
37  * to the University of California by American Telephone and Telegraph
38  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
39  * the permission of UNIX System Laboratories, Inc.
40  *
41  * Redistribution and use in source and binary forms, with or without
42  * modification, are permitted provided that the following conditions
43  * are met:
44  * 1. Redistributions of source code must retain the above copyright
45  *    notice, this list of conditions and the following disclaimer.
46  * 2. Redistributions in binary form must reproduce the above copyright
47  *    notice, this list of conditions and the following disclaimer in the
48  *    documentation and/or other materials provided with the distribution.
49  * 3. Neither the name of the University nor the names of its contributors
50  *    may be used to endorse or promote products derived from this software
51  *    without specific prior written permission.
52  *
53  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
54  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
55  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
56  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
57  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
58  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
59  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
60  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
61  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
62  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
63  * SUCH DAMAGE.
64  *
65  *	@(#)vfs_bio.c	8.6 (Berkeley) 1/11/94
66  */
67 
68 /*-
69  * Copyright (c) 1994 Christopher G. Demetriou
70  *
71  * Redistribution and use in source and binary forms, with or without
72  * modification, are permitted provided that the following conditions
73  * are met:
74  * 1. Redistributions of source code must retain the above copyright
75  *    notice, this list of conditions and the following disclaimer.
76  * 2. Redistributions in binary form must reproduce the above copyright
77  *    notice, this list of conditions and the following disclaimer in the
78  *    documentation and/or other materials provided with the distribution.
79  * 3. All advertising materials mentioning features or use of this software
80  *    must display the following acknowledgement:
81  *	This product includes software developed by the University of
82  *	California, Berkeley and its contributors.
83  * 4. Neither the name of the University nor the names of its contributors
84  *    may be used to endorse or promote products derived from this software
85  *    without specific prior written permission.
86  *
87  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
88  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
89  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
90  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
91  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
92  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
93  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
94  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
95  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
96  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
97  * SUCH DAMAGE.
98  *
99  *	@(#)vfs_bio.c	8.6 (Berkeley) 1/11/94
100  */
101 
102 /*
103  * The buffer cache subsystem.
104  *
105  * Some references:
106  *	Bach: The Design of the UNIX Operating System (Prentice Hall, 1986)
107  *	Leffler, et al.: The Design and Implementation of the 4.3BSD
108  *		UNIX Operating System (Addison Welley, 1989)
109  *
110  * Locking
111  *
112  * There are three locks:
113  * - bufcache_lock: protects global buffer cache state.
114  * - BC_BUSY: a long term per-buffer lock.
115  * - buf_t::b_objlock: lock on completion (biowait vs biodone).
116  *
117  * For buffers associated with vnodes (a most common case) b_objlock points
118  * to the vnode_t::v_interlock.  Otherwise, it points to generic buffer_lock.
119  *
120  * Lock order:
121  *	bufcache_lock ->
122  *		buf_t::b_objlock
123  */
124 
125 #include <sys/cdefs.h>
126 __KERNEL_RCSID(0, "$NetBSD: vfs_bio.c,v 1.230 2011/06/12 03:35:56 rmind Exp $");
127 
128 #include "opt_bufcache.h"
129 
130 #include <sys/param.h>
131 #include <sys/systm.h>
132 #include <sys/kernel.h>
133 #include <sys/proc.h>
134 #include <sys/buf.h>
135 #include <sys/vnode.h>
136 #include <sys/mount.h>
137 #include <sys/resourcevar.h>
138 #include <sys/sysctl.h>
139 #include <sys/conf.h>
140 #include <sys/kauth.h>
141 #include <sys/fstrans.h>
142 #include <sys/intr.h>
143 #include <sys/cpu.h>
144 #include <sys/wapbl.h>
145 
146 #include <uvm/uvm.h>	/* extern struct uvm uvm */
147 
148 #include <miscfs/specfs/specdev.h>
149 
150 #ifndef	BUFPAGES
151 # define BUFPAGES 0
152 #endif
153 
154 #ifdef BUFCACHE
155 # if (BUFCACHE < 5) || (BUFCACHE > 95)
156 #  error BUFCACHE is not between 5 and 95
157 # endif
158 #else
159 # define BUFCACHE 15
160 #endif
161 
162 u_int	nbuf;			/* desired number of buffer headers */
163 u_int	bufpages = BUFPAGES;	/* optional hardwired count */
164 u_int	bufcache = BUFCACHE;	/* max % of RAM to use for buffer cache */
165 
166 /* Function prototypes */
167 struct bqueue;
168 
169 static void buf_setwm(void);
170 static int buf_trim(void);
171 static void *bufpool_page_alloc(struct pool *, int);
172 static void bufpool_page_free(struct pool *, void *);
173 static buf_t *bio_doread(struct vnode *, daddr_t, int,
174     kauth_cred_t, int);
175 static buf_t *getnewbuf(int, int, int);
176 static int buf_lotsfree(void);
177 static int buf_canrelease(void);
178 static u_long buf_mempoolidx(u_long);
179 static u_long buf_roundsize(u_long);
180 static void *buf_alloc(size_t);
181 static void buf_mrelease(void *, size_t);
182 static void binsheadfree(buf_t *, struct bqueue *);
183 static void binstailfree(buf_t *, struct bqueue *);
184 #ifdef DEBUG
185 static int checkfreelist(buf_t *, struct bqueue *, int);
186 #endif
187 static void biointr(void *);
188 static void biodone2(buf_t *);
189 static void bref(buf_t *);
190 static void brele(buf_t *);
191 static void sysctl_kern_buf_setup(void);
192 static void sysctl_vm_buf_setup(void);
193 
194 /*
195  * Definitions for the buffer hash lists.
196  */
197 #define	BUFHASH(dvp, lbn)	\
198 	(&bufhashtbl[(((long)(dvp) >> 8) + (int)(lbn)) & bufhash])
199 LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash;
200 u_long	bufhash;
201 struct bqueue bufqueues[BQUEUES];
202 
203 static kcondvar_t needbuffer_cv;
204 
205 /*
206  * Buffer queue lock.
207  */
208 kmutex_t bufcache_lock;
209 kmutex_t buffer_lock;
210 
211 /* Software ISR for completed transfers. */
212 static void *biodone_sih;
213 
214 /* Buffer pool for I/O buffers. */
215 static pool_cache_t buf_cache;
216 static pool_cache_t bufio_cache;
217 
218 /* XXX - somewhat gross.. */
219 #if MAXBSIZE == 0x2000
220 #define NMEMPOOLS 5
221 #elif MAXBSIZE == 0x4000
222 #define NMEMPOOLS 6
223 #elif MAXBSIZE == 0x8000
224 #define NMEMPOOLS 7
225 #else
226 #define NMEMPOOLS 8
227 #endif
228 
229 #define MEMPOOL_INDEX_OFFSET 9	/* smallest pool is 512 bytes */
230 #if (1 << (NMEMPOOLS + MEMPOOL_INDEX_OFFSET - 1)) != MAXBSIZE
231 #error update vfs_bio buffer memory parameters
232 #endif
233 
234 /* Buffer memory pools */
235 static struct pool bmempools[NMEMPOOLS];
236 
237 static struct vm_map *buf_map;
238 
239 /*
240  * Buffer memory pool allocator.
241  */
242 static void *
243 bufpool_page_alloc(struct pool *pp, int flags)
244 {
245 
246 	return (void *)uvm_km_alloc(buf_map,
247 	    MAXBSIZE, MAXBSIZE,
248 	    ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT | UVM_KMF_TRYLOCK)
249 	    | UVM_KMF_WIRED);
250 }
251 
252 static void
253 bufpool_page_free(struct pool *pp, void *v)
254 {
255 
256 	uvm_km_free(buf_map, (vaddr_t)v, MAXBSIZE, UVM_KMF_WIRED);
257 }
258 
259 static struct pool_allocator bufmempool_allocator = {
260 	.pa_alloc = bufpool_page_alloc,
261 	.pa_free = bufpool_page_free,
262 	.pa_pagesz = MAXBSIZE,
263 };
264 
265 /* Buffer memory management variables */
266 u_long bufmem_valimit;
267 u_long bufmem_hiwater;
268 u_long bufmem_lowater;
269 u_long bufmem;
270 
271 /*
272  * MD code can call this to set a hard limit on the amount
273  * of virtual memory used by the buffer cache.
274  */
275 int
276 buf_setvalimit(vsize_t sz)
277 {
278 
279 	/* We need to accommodate at least NMEMPOOLS of MAXBSIZE each */
280 	if (sz < NMEMPOOLS * MAXBSIZE)
281 		return EINVAL;
282 
283 	bufmem_valimit = sz;
284 	return 0;
285 }
286 
287 static void
288 buf_setwm(void)
289 {
290 
291 	bufmem_hiwater = buf_memcalc();
292 	/* lowater is approx. 2% of memory (with bufcache = 15) */
293 #define	BUFMEM_WMSHIFT	3
294 #define	BUFMEM_HIWMMIN	(64 * 1024 << BUFMEM_WMSHIFT)
295 	if (bufmem_hiwater < BUFMEM_HIWMMIN)
296 		/* Ensure a reasonable minimum value */
297 		bufmem_hiwater = BUFMEM_HIWMMIN;
298 	bufmem_lowater = bufmem_hiwater >> BUFMEM_WMSHIFT;
299 }
300 
301 #ifdef DEBUG
302 int debug_verify_freelist = 0;
303 static int
304 checkfreelist(buf_t *bp, struct bqueue *dp, int ison)
305 {
306 	buf_t *b;
307 
308 	if (!debug_verify_freelist)
309 		return 1;
310 
311 	TAILQ_FOREACH(b, &dp->bq_queue, b_freelist) {
312 		if (b == bp)
313 			return ison ? 1 : 0;
314 	}
315 
316 	return ison ? 0 : 1;
317 }
318 #endif
319 
320 /*
321  * Insq/Remq for the buffer hash lists.
322  * Call with buffer queue locked.
323  */
324 static void
325 binsheadfree(buf_t *bp, struct bqueue *dp)
326 {
327 
328 	KASSERT(mutex_owned(&bufcache_lock));
329 	KASSERT(bp->b_freelistindex == -1);
330 	TAILQ_INSERT_HEAD(&dp->bq_queue, bp, b_freelist);
331 	dp->bq_bytes += bp->b_bufsize;
332 	bp->b_freelistindex = dp - bufqueues;
333 }
334 
335 static void
336 binstailfree(buf_t *bp, struct bqueue *dp)
337 {
338 
339 	KASSERT(mutex_owned(&bufcache_lock));
340 	KASSERT(bp->b_freelistindex == -1);
341 	TAILQ_INSERT_TAIL(&dp->bq_queue, bp, b_freelist);
342 	dp->bq_bytes += bp->b_bufsize;
343 	bp->b_freelistindex = dp - bufqueues;
344 }
345 
346 void
347 bremfree(buf_t *bp)
348 {
349 	struct bqueue *dp;
350 	int bqidx = bp->b_freelistindex;
351 
352 	KASSERT(mutex_owned(&bufcache_lock));
353 
354 	KASSERT(bqidx != -1);
355 	dp = &bufqueues[bqidx];
356 	KDASSERT(checkfreelist(bp, dp, 1));
357 	KASSERT(dp->bq_bytes >= bp->b_bufsize);
358 	TAILQ_REMOVE(&dp->bq_queue, bp, b_freelist);
359 	dp->bq_bytes -= bp->b_bufsize;
360 
361 	/* For the sysctl helper. */
362 	if (bp == dp->bq_marker)
363 		dp->bq_marker = NULL;
364 
365 #if defined(DIAGNOSTIC)
366 	bp->b_freelistindex = -1;
367 #endif /* defined(DIAGNOSTIC) */
368 }
369 
370 /*
371  * Add a reference to an buffer structure that came from buf_cache.
372  */
373 static inline void
374 bref(buf_t *bp)
375 {
376 
377 	KASSERT(mutex_owned(&bufcache_lock));
378 	KASSERT(bp->b_refcnt > 0);
379 
380 	bp->b_refcnt++;
381 }
382 
383 /*
384  * Free an unused buffer structure that came from buf_cache.
385  */
386 static inline void
387 brele(buf_t *bp)
388 {
389 
390 	KASSERT(mutex_owned(&bufcache_lock));
391 	KASSERT(bp->b_refcnt > 0);
392 
393 	if (bp->b_refcnt-- == 1) {
394 		buf_destroy(bp);
395 #ifdef DEBUG
396 		memset((char *)bp, 0, sizeof(*bp));
397 #endif
398 		pool_cache_put(buf_cache, bp);
399 	}
400 }
401 
402 /*
403  * note that for some ports this is used by pmap bootstrap code to
404  * determine kva size.
405  */
406 u_long
407 buf_memcalc(void)
408 {
409 	u_long n;
410 
411 	/*
412 	 * Determine the upper bound of memory to use for buffers.
413 	 *
414 	 *	- If bufpages is specified, use that as the number
415 	 *	  pages.
416 	 *
417 	 *	- Otherwise, use bufcache as the percentage of
418 	 *	  physical memory.
419 	 */
420 	if (bufpages != 0) {
421 		n = bufpages;
422 	} else {
423 		if (bufcache < 5) {
424 			printf("forcing bufcache %d -> 5", bufcache);
425 			bufcache = 5;
426 		}
427 		if (bufcache > 95) {
428 			printf("forcing bufcache %d -> 95", bufcache);
429 			bufcache = 95;
430 		}
431 		n = calc_cache_size(buf_map, bufcache,
432 		    (buf_map != kernel_map) ? 100 : BUFCACHE_VA_MAXPCT)
433 		    / PAGE_SIZE;
434 	}
435 
436 	n <<= PAGE_SHIFT;
437 	if (bufmem_valimit != 0 && n > bufmem_valimit)
438 		n = bufmem_valimit;
439 
440 	return (n);
441 }
442 
443 /*
444  * Initialize buffers and hash links for buffers.
445  */
446 void
447 bufinit(void)
448 {
449 	struct bqueue *dp;
450 	int use_std;
451 	u_int i;
452 
453 	mutex_init(&bufcache_lock, MUTEX_DEFAULT, IPL_NONE);
454 	mutex_init(&buffer_lock, MUTEX_DEFAULT, IPL_NONE);
455 	cv_init(&needbuffer_cv, "needbuf");
456 
457 	if (bufmem_valimit != 0) {
458 		vaddr_t minaddr = 0, maxaddr;
459 		buf_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
460 					  bufmem_valimit, 0, false, 0);
461 		if (buf_map == NULL)
462 			panic("bufinit: cannot allocate submap");
463 	} else
464 		buf_map = kernel_map;
465 
466 	/*
467 	 * Initialize buffer cache memory parameters.
468 	 */
469 	bufmem = 0;
470 	buf_setwm();
471 
472 	/* On "small" machines use small pool page sizes where possible */
473 	use_std = (physmem < atop(16*1024*1024));
474 
475 	/*
476 	 * Also use them on systems that can map the pool pages using
477 	 * a direct-mapped segment.
478 	 */
479 #ifdef PMAP_MAP_POOLPAGE
480 	use_std = 1;
481 #endif
482 
483 	buf_cache = pool_cache_init(sizeof(buf_t), 0, 0, 0,
484 	    "bufpl", NULL, IPL_SOFTBIO, NULL, NULL, NULL);
485 	bufio_cache = pool_cache_init(sizeof(buf_t), 0, 0, 0,
486 	    "biopl", NULL, IPL_BIO, NULL, NULL, NULL);
487 
488 	bufmempool_allocator.pa_backingmap = buf_map;
489 	for (i = 0; i < NMEMPOOLS; i++) {
490 		struct pool_allocator *pa;
491 		struct pool *pp = &bmempools[i];
492 		u_int size = 1 << (i + MEMPOOL_INDEX_OFFSET);
493 		char *name = kmem_alloc(8, KM_SLEEP); /* XXX: never freed */
494 		if (__predict_true(size >= 1024))
495 			(void)snprintf(name, 8, "buf%dk", size / 1024);
496 		else
497 			(void)snprintf(name, 8, "buf%db", size);
498 		pa = (size <= PAGE_SIZE && use_std)
499 			? &pool_allocator_nointr
500 			: &bufmempool_allocator;
501 		pool_init(pp, size, 0, 0, 0, name, pa, IPL_NONE);
502 		pool_setlowat(pp, 1);
503 		pool_sethiwat(pp, 1);
504 	}
505 
506 	/* Initialize the buffer queues */
507 	for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++) {
508 		TAILQ_INIT(&dp->bq_queue);
509 		dp->bq_bytes = 0;
510 	}
511 
512 	/*
513 	 * Estimate hash table size based on the amount of memory we
514 	 * intend to use for the buffer cache. The average buffer
515 	 * size is dependent on our clients (i.e. filesystems).
516 	 *
517 	 * For now, use an empirical 3K per buffer.
518 	 */
519 	nbuf = (bufmem_hiwater / 1024) / 3;
520 	bufhashtbl = hashinit(nbuf, HASH_LIST, true, &bufhash);
521 
522 	sysctl_kern_buf_setup();
523 	sysctl_vm_buf_setup();
524 }
525 
526 void
527 bufinit2(void)
528 {
529 
530 	biodone_sih = softint_establish(SOFTINT_BIO | SOFTINT_MPSAFE, biointr,
531 	    NULL);
532 	if (biodone_sih == NULL)
533 		panic("bufinit2: can't establish soft interrupt");
534 }
535 
536 static int
537 buf_lotsfree(void)
538 {
539 	int try, thresh;
540 
541 	/* Always allocate if less than the low water mark. */
542 	if (bufmem < bufmem_lowater)
543 		return 1;
544 
545 	/* Never allocate if greater than the high water mark. */
546 	if (bufmem > bufmem_hiwater)
547 		return 0;
548 
549 	/* If there's anything on the AGE list, it should be eaten. */
550 	if (TAILQ_FIRST(&bufqueues[BQ_AGE].bq_queue) != NULL)
551 		return 0;
552 
553 	/*
554 	 * The probabily of getting a new allocation is inversely
555 	 * proportional to the current size of the cache, using
556 	 * a granularity of 16 steps.
557 	 */
558 	try = random() & 0x0000000fL;
559 
560 	/* Don't use "16 * bufmem" here to avoid a 32-bit overflow. */
561 	thresh = (bufmem - bufmem_lowater) /
562 	    ((bufmem_hiwater - bufmem_lowater) / 16);
563 
564 	if (try >= thresh)
565 		return 1;
566 
567 	/* Otherwise don't allocate. */
568 	return 0;
569 }
570 
571 /*
572  * Return estimate of bytes we think need to be
573  * released to help resolve low memory conditions.
574  *
575  * => called with bufcache_lock held.
576  */
577 static int
578 buf_canrelease(void)
579 {
580 	int pagedemand, ninvalid = 0;
581 
582 	KASSERT(mutex_owned(&bufcache_lock));
583 
584 	if (bufmem < bufmem_lowater)
585 		return 0;
586 
587 	if (bufmem > bufmem_hiwater)
588 		return bufmem - bufmem_hiwater;
589 
590 	ninvalid += bufqueues[BQ_AGE].bq_bytes;
591 
592 	pagedemand = uvmexp.freetarg - uvmexp.free;
593 	if (pagedemand < 0)
594 		return ninvalid;
595 	return MAX(ninvalid, MIN(2 * MAXBSIZE,
596 	    MIN((bufmem - bufmem_lowater) / 16, pagedemand * PAGE_SIZE)));
597 }
598 
599 /*
600  * Buffer memory allocation helper functions
601  */
602 static u_long
603 buf_mempoolidx(u_long size)
604 {
605 	u_int n = 0;
606 
607 	size -= 1;
608 	size >>= MEMPOOL_INDEX_OFFSET;
609 	while (size) {
610 		size >>= 1;
611 		n += 1;
612 	}
613 	if (n >= NMEMPOOLS)
614 		panic("buf mem pool index %d", n);
615 	return n;
616 }
617 
618 static u_long
619 buf_roundsize(u_long size)
620 {
621 	/* Round up to nearest power of 2 */
622 	return (1 << (buf_mempoolidx(size) + MEMPOOL_INDEX_OFFSET));
623 }
624 
625 static void *
626 buf_alloc(size_t size)
627 {
628 	u_int n = buf_mempoolidx(size);
629 	void *addr;
630 
631 	while (1) {
632 		addr = pool_get(&bmempools[n], PR_NOWAIT);
633 		if (addr != NULL)
634 			break;
635 
636 		/* No memory, see if we can free some. If so, try again */
637 		mutex_enter(&bufcache_lock);
638 		if (buf_drain(1) > 0) {
639 			mutex_exit(&bufcache_lock);
640 			continue;
641 		}
642 
643 		if (curlwp == uvm.pagedaemon_lwp) {
644 			mutex_exit(&bufcache_lock);
645 			return NULL;
646 		}
647 
648 		/* Wait for buffers to arrive on the LRU queue */
649 		cv_timedwait(&needbuffer_cv, &bufcache_lock, hz / 4);
650 		mutex_exit(&bufcache_lock);
651 	}
652 
653 	return addr;
654 }
655 
656 static void
657 buf_mrelease(void *addr, size_t size)
658 {
659 
660 	pool_put(&bmempools[buf_mempoolidx(size)], addr);
661 }
662 
663 /*
664  * bread()/breadn() helper.
665  */
666 static buf_t *
667 bio_doread(struct vnode *vp, daddr_t blkno, int size, kauth_cred_t cred,
668     int async)
669 {
670 	buf_t *bp;
671 	struct mount *mp;
672 
673 	bp = getblk(vp, blkno, size, 0, 0);
674 
675 #ifdef DIAGNOSTIC
676 	if (bp == NULL) {
677 		panic("bio_doread: no such buf");
678 	}
679 #endif
680 
681 	/*
682 	 * If buffer does not have data valid, start a read.
683 	 * Note that if buffer is BC_INVAL, getblk() won't return it.
684 	 * Therefore, it's valid if its I/O has completed or been delayed.
685 	 */
686 	if (!ISSET(bp->b_oflags, (BO_DONE | BO_DELWRI))) {
687 		/* Start I/O for the buffer. */
688 		SET(bp->b_flags, B_READ | async);
689 		if (async)
690 			BIO_SETPRIO(bp, BPRIO_TIMELIMITED);
691 		else
692 			BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
693 		VOP_STRATEGY(vp, bp);
694 
695 		/* Pay for the read. */
696 		curlwp->l_ru.ru_inblock++;
697 	} else if (async)
698 		brelse(bp, 0);
699 
700 	if (vp->v_type == VBLK)
701 		mp = vp->v_specmountpoint;
702 	else
703 		mp = vp->v_mount;
704 
705 	/*
706 	 * Collect statistics on synchronous and asynchronous reads.
707 	 * Reads from block devices are charged to their associated
708 	 * filesystem (if any).
709 	 */
710 	if (mp != NULL) {
711 		if (async == 0)
712 			mp->mnt_stat.f_syncreads++;
713 		else
714 			mp->mnt_stat.f_asyncreads++;
715 	}
716 
717 	return (bp);
718 }
719 
720 /*
721  * Read a disk block.
722  * This algorithm described in Bach (p.54).
723  */
724 int
725 bread(struct vnode *vp, daddr_t blkno, int size, kauth_cred_t cred,
726     int flags, buf_t **bpp)
727 {
728 	buf_t *bp;
729 	int error;
730 
731 	/* Get buffer for block. */
732 	bp = *bpp = bio_doread(vp, blkno, size, cred, 0);
733 
734 	/* Wait for the read to complete, and return result. */
735 	error = biowait(bp);
736 	if (error == 0 && (flags & B_MODIFY) != 0)
737 		error = fscow_run(bp, true);
738 
739 	return error;
740 }
741 
742 /*
743  * Read-ahead multiple disk blocks. The first is sync, the rest async.
744  * Trivial modification to the breada algorithm presented in Bach (p.55).
745  */
746 int
747 breadn(struct vnode *vp, daddr_t blkno, int size, daddr_t *rablks,
748     int *rasizes, int nrablks, kauth_cred_t cred, int flags, buf_t **bpp)
749 {
750 	buf_t *bp;
751 	int error, i;
752 
753 	bp = *bpp = bio_doread(vp, blkno, size, cred, 0);
754 
755 	/*
756 	 * For each of the read-ahead blocks, start a read, if necessary.
757 	 */
758 	mutex_enter(&bufcache_lock);
759 	for (i = 0; i < nrablks; i++) {
760 		/* If it's in the cache, just go on to next one. */
761 		if (incore(vp, rablks[i]))
762 			continue;
763 
764 		/* Get a buffer for the read-ahead block */
765 		mutex_exit(&bufcache_lock);
766 		(void) bio_doread(vp, rablks[i], rasizes[i], cred, B_ASYNC);
767 		mutex_enter(&bufcache_lock);
768 	}
769 	mutex_exit(&bufcache_lock);
770 
771 	/* Otherwise, we had to start a read for it; wait until it's valid. */
772 	error = biowait(bp);
773 	if (error == 0 && (flags & B_MODIFY) != 0)
774 		error = fscow_run(bp, true);
775 	return error;
776 }
777 
778 /*
779  * Block write.  Described in Bach (p.56)
780  */
781 int
782 bwrite(buf_t *bp)
783 {
784 	int rv, sync, wasdelayed;
785 	struct vnode *vp;
786 	struct mount *mp;
787 
788 	KASSERT(ISSET(bp->b_cflags, BC_BUSY));
789 	KASSERT(!cv_has_waiters(&bp->b_done));
790 
791 	vp = bp->b_vp;
792 	if (vp != NULL) {
793 		KASSERT(bp->b_objlock == vp->v_interlock);
794 		if (vp->v_type == VBLK)
795 			mp = vp->v_specmountpoint;
796 		else
797 			mp = vp->v_mount;
798 	} else {
799 		mp = NULL;
800 	}
801 
802 	if (mp && mp->mnt_wapbl) {
803 		if (bp->b_iodone != mp->mnt_wapbl_op->wo_wapbl_biodone) {
804 			bdwrite(bp);
805 			return 0;
806 		}
807 	}
808 
809 	/*
810 	 * Remember buffer type, to switch on it later.  If the write was
811 	 * synchronous, but the file system was mounted with MNT_ASYNC,
812 	 * convert it to a delayed write.
813 	 * XXX note that this relies on delayed tape writes being converted
814 	 * to async, not sync writes (which is safe, but ugly).
815 	 */
816 	sync = !ISSET(bp->b_flags, B_ASYNC);
817 	if (sync && mp != NULL && ISSET(mp->mnt_flag, MNT_ASYNC)) {
818 		bdwrite(bp);
819 		return (0);
820 	}
821 
822 	/*
823 	 * Collect statistics on synchronous and asynchronous writes.
824 	 * Writes to block devices are charged to their associated
825 	 * filesystem (if any).
826 	 */
827 	if (mp != NULL) {
828 		if (sync)
829 			mp->mnt_stat.f_syncwrites++;
830 		else
831 			mp->mnt_stat.f_asyncwrites++;
832 	}
833 
834 	/*
835 	 * Pay for the I/O operation and make sure the buf is on the correct
836 	 * vnode queue.
837 	 */
838 	bp->b_error = 0;
839 	wasdelayed = ISSET(bp->b_oflags, BO_DELWRI);
840 	CLR(bp->b_flags, B_READ);
841 	if (wasdelayed) {
842 		mutex_enter(&bufcache_lock);
843 		mutex_enter(bp->b_objlock);
844 		CLR(bp->b_oflags, BO_DONE | BO_DELWRI);
845 		reassignbuf(bp, bp->b_vp);
846 		mutex_exit(&bufcache_lock);
847 	} else {
848 		curlwp->l_ru.ru_oublock++;
849 		mutex_enter(bp->b_objlock);
850 		CLR(bp->b_oflags, BO_DONE | BO_DELWRI);
851 	}
852 	if (vp != NULL)
853 		vp->v_numoutput++;
854 	mutex_exit(bp->b_objlock);
855 
856 	/* Initiate disk write. */
857 	if (sync)
858 		BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
859 	else
860 		BIO_SETPRIO(bp, BPRIO_TIMELIMITED);
861 
862 	VOP_STRATEGY(vp, bp);
863 
864 	if (sync) {
865 		/* If I/O was synchronous, wait for it to complete. */
866 		rv = biowait(bp);
867 
868 		/* Release the buffer. */
869 		brelse(bp, 0);
870 
871 		return (rv);
872 	} else {
873 		return (0);
874 	}
875 }
876 
877 int
878 vn_bwrite(void *v)
879 {
880 	struct vop_bwrite_args *ap = v;
881 
882 	return (bwrite(ap->a_bp));
883 }
884 
885 /*
886  * Delayed write.
887  *
888  * The buffer is marked dirty, but is not queued for I/O.
889  * This routine should be used when the buffer is expected
890  * to be modified again soon, typically a small write that
891  * partially fills a buffer.
892  *
893  * NB: magnetic tapes cannot be delayed; they must be
894  * written in the order that the writes are requested.
895  *
896  * Described in Leffler, et al. (pp. 208-213).
897  */
898 void
899 bdwrite(buf_t *bp)
900 {
901 
902 	KASSERT(bp->b_vp == NULL || bp->b_vp->v_tag != VT_UFS ||
903 	    bp->b_vp->v_type == VBLK || ISSET(bp->b_flags, B_COWDONE));
904 	KASSERT(ISSET(bp->b_cflags, BC_BUSY));
905 	KASSERT(!cv_has_waiters(&bp->b_done));
906 
907 	/* If this is a tape block, write the block now. */
908 	if (bdev_type(bp->b_dev) == D_TAPE) {
909 		bawrite(bp);
910 		return;
911 	}
912 
913 	if (wapbl_vphaswapbl(bp->b_vp)) {
914 		struct mount *mp = wapbl_vptomp(bp->b_vp);
915 
916 		if (bp->b_iodone != mp->mnt_wapbl_op->wo_wapbl_biodone) {
917 			WAPBL_ADD_BUF(mp, bp);
918 		}
919 	}
920 
921 	/*
922 	 * If the block hasn't been seen before:
923 	 *	(1) Mark it as having been seen,
924 	 *	(2) Charge for the write,
925 	 *	(3) Make sure it's on its vnode's correct block list.
926 	 */
927 	KASSERT(bp->b_vp == NULL || bp->b_objlock == bp->b_vp->v_interlock);
928 
929 	if (!ISSET(bp->b_oflags, BO_DELWRI)) {
930 		mutex_enter(&bufcache_lock);
931 		mutex_enter(bp->b_objlock);
932 		SET(bp->b_oflags, BO_DELWRI);
933 		curlwp->l_ru.ru_oublock++;
934 		reassignbuf(bp, bp->b_vp);
935 		mutex_exit(&bufcache_lock);
936 	} else {
937 		mutex_enter(bp->b_objlock);
938 	}
939 	/* Otherwise, the "write" is done, so mark and release the buffer. */
940 	CLR(bp->b_oflags, BO_DONE);
941 	mutex_exit(bp->b_objlock);
942 
943 	brelse(bp, 0);
944 }
945 
946 /*
947  * Asynchronous block write; just an asynchronous bwrite().
948  */
949 void
950 bawrite(buf_t *bp)
951 {
952 
953 	KASSERT(ISSET(bp->b_cflags, BC_BUSY));
954 
955 	SET(bp->b_flags, B_ASYNC);
956 	VOP_BWRITE(bp);
957 }
958 
959 /*
960  * Release a buffer on to the free lists.
961  * Described in Bach (p. 46).
962  */
963 void
964 brelsel(buf_t *bp, int set)
965 {
966 	struct bqueue *bufq;
967 	struct vnode *vp;
968 
969 	KASSERT(mutex_owned(&bufcache_lock));
970 	KASSERT(!cv_has_waiters(&bp->b_done));
971 	KASSERT(bp->b_refcnt > 0);
972 
973 	SET(bp->b_cflags, set);
974 
975 	KASSERT(ISSET(bp->b_cflags, BC_BUSY));
976 	KASSERT(bp->b_iodone == NULL);
977 
978 	/* Wake up any processes waiting for any buffer to become free. */
979 	cv_signal(&needbuffer_cv);
980 
981 	/* Wake up any proceeses waiting for _this_ buffer to become */
982 	if (ISSET(bp->b_cflags, BC_WANTED))
983 		CLR(bp->b_cflags, BC_WANTED|BC_AGE);
984 
985 	/* If it's clean clear the copy-on-write flag. */
986 	if (ISSET(bp->b_flags, B_COWDONE)) {
987 		mutex_enter(bp->b_objlock);
988 		if (!ISSET(bp->b_oflags, BO_DELWRI))
989 			CLR(bp->b_flags, B_COWDONE);
990 		mutex_exit(bp->b_objlock);
991 	}
992 
993 	/*
994 	 * Determine which queue the buffer should be on, then put it there.
995 	 */
996 
997 	/* If it's locked, don't report an error; try again later. */
998 	if (ISSET(bp->b_flags, B_LOCKED))
999 		bp->b_error = 0;
1000 
1001 	/* If it's not cacheable, or an error, mark it invalid. */
1002 	if (ISSET(bp->b_cflags, BC_NOCACHE) || bp->b_error != 0)
1003 		SET(bp->b_cflags, BC_INVAL);
1004 
1005 	if (ISSET(bp->b_cflags, BC_VFLUSH)) {
1006 		/*
1007 		 * This is a delayed write buffer that was just flushed to
1008 		 * disk.  It is still on the LRU queue.  If it's become
1009 		 * invalid, then we need to move it to a different queue;
1010 		 * otherwise leave it in its current position.
1011 		 */
1012 		CLR(bp->b_cflags, BC_VFLUSH);
1013 		if (!ISSET(bp->b_cflags, BC_INVAL|BC_AGE) &&
1014 		    !ISSET(bp->b_flags, B_LOCKED) && bp->b_error == 0) {
1015 			KDASSERT(checkfreelist(bp, &bufqueues[BQ_LRU], 1));
1016 			goto already_queued;
1017 		} else {
1018 			bremfree(bp);
1019 		}
1020 	}
1021 
1022 	KDASSERT(checkfreelist(bp, &bufqueues[BQ_AGE], 0));
1023 	KDASSERT(checkfreelist(bp, &bufqueues[BQ_LRU], 0));
1024 	KDASSERT(checkfreelist(bp, &bufqueues[BQ_LOCKED], 0));
1025 
1026 	if ((bp->b_bufsize <= 0) || ISSET(bp->b_cflags, BC_INVAL)) {
1027 		/*
1028 		 * If it's invalid or empty, dissociate it from its vnode
1029 		 * and put on the head of the appropriate queue.
1030 		 */
1031 		if (ISSET(bp->b_flags, B_LOCKED)) {
1032 			if (wapbl_vphaswapbl(vp = bp->b_vp)) {
1033 				struct mount *mp = wapbl_vptomp(vp);
1034 
1035 				KASSERT(bp->b_iodone
1036 				    != mp->mnt_wapbl_op->wo_wapbl_biodone);
1037 				WAPBL_REMOVE_BUF(mp, bp);
1038 			}
1039 		}
1040 
1041 		mutex_enter(bp->b_objlock);
1042 		CLR(bp->b_oflags, BO_DONE|BO_DELWRI);
1043 		if ((vp = bp->b_vp) != NULL) {
1044 			KASSERT(bp->b_objlock == vp->v_interlock);
1045 			reassignbuf(bp, bp->b_vp);
1046 			brelvp(bp);
1047 			mutex_exit(vp->v_interlock);
1048 		} else {
1049 			KASSERT(bp->b_objlock == &buffer_lock);
1050 			mutex_exit(bp->b_objlock);
1051 		}
1052 
1053 		if (bp->b_bufsize <= 0)
1054 			/* no data */
1055 			goto already_queued;
1056 		else
1057 			/* invalid data */
1058 			bufq = &bufqueues[BQ_AGE];
1059 		binsheadfree(bp, bufq);
1060 	} else  {
1061 		/*
1062 		 * It has valid data.  Put it on the end of the appropriate
1063 		 * queue, so that it'll stick around for as long as possible.
1064 		 * If buf is AGE, but has dependencies, must put it on last
1065 		 * bufqueue to be scanned, ie LRU. This protects against the
1066 		 * livelock where BQ_AGE only has buffers with dependencies,
1067 		 * and we thus never get to the dependent buffers in BQ_LRU.
1068 		 */
1069 		if (ISSET(bp->b_flags, B_LOCKED)) {
1070 			/* locked in core */
1071 			bufq = &bufqueues[BQ_LOCKED];
1072 		} else if (!ISSET(bp->b_cflags, BC_AGE)) {
1073 			/* valid data */
1074 			bufq = &bufqueues[BQ_LRU];
1075 		} else {
1076 			/* stale but valid data */
1077 			bufq = &bufqueues[BQ_AGE];
1078 		}
1079 		binstailfree(bp, bufq);
1080 	}
1081 already_queued:
1082 	/* Unlock the buffer. */
1083 	CLR(bp->b_cflags, BC_AGE|BC_BUSY|BC_NOCACHE);
1084 	CLR(bp->b_flags, B_ASYNC);
1085 	cv_broadcast(&bp->b_busy);
1086 
1087 	if (bp->b_bufsize <= 0)
1088 		brele(bp);
1089 }
1090 
1091 void
1092 brelse(buf_t *bp, int set)
1093 {
1094 
1095 	mutex_enter(&bufcache_lock);
1096 	brelsel(bp, set);
1097 	mutex_exit(&bufcache_lock);
1098 }
1099 
1100 /*
1101  * Determine if a block is in the cache.
1102  * Just look on what would be its hash chain.  If it's there, return
1103  * a pointer to it, unless it's marked invalid.  If it's marked invalid,
1104  * we normally don't return the buffer, unless the caller explicitly
1105  * wants us to.
1106  */
1107 buf_t *
1108 incore(struct vnode *vp, daddr_t blkno)
1109 {
1110 	buf_t *bp;
1111 
1112 	KASSERT(mutex_owned(&bufcache_lock));
1113 
1114 	/* Search hash chain */
1115 	LIST_FOREACH(bp, BUFHASH(vp, blkno), b_hash) {
1116 		if (bp->b_lblkno == blkno && bp->b_vp == vp &&
1117 		    !ISSET(bp->b_cflags, BC_INVAL)) {
1118 		    	KASSERT(bp->b_objlock == vp->v_interlock);
1119 		    	return (bp);
1120 		}
1121 	}
1122 
1123 	return (NULL);
1124 }
1125 
1126 /*
1127  * Get a block of requested size that is associated with
1128  * a given vnode and block offset. If it is found in the
1129  * block cache, mark it as having been found, make it busy
1130  * and return it. Otherwise, return an empty block of the
1131  * correct size. It is up to the caller to insure that the
1132  * cached blocks be of the correct size.
1133  */
1134 buf_t *
1135 getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, int slptimeo)
1136 {
1137 	int err, preserve;
1138 	buf_t *bp;
1139 
1140 	mutex_enter(&bufcache_lock);
1141  loop:
1142 	bp = incore(vp, blkno);
1143 	if (bp != NULL) {
1144 		err = bbusy(bp, ((slpflag & PCATCH) != 0), slptimeo, NULL);
1145 		if (err != 0) {
1146 			if (err == EPASSTHROUGH)
1147 				goto loop;
1148 			mutex_exit(&bufcache_lock);
1149 			return (NULL);
1150 		}
1151 		KASSERT(!cv_has_waiters(&bp->b_done));
1152 #ifdef DIAGNOSTIC
1153 		if (ISSET(bp->b_oflags, BO_DONE|BO_DELWRI) &&
1154 		    bp->b_bcount < size && vp->v_type != VBLK)
1155 			panic("getblk: block size invariant failed");
1156 #endif
1157 		bremfree(bp);
1158 		preserve = 1;
1159 	} else {
1160 		if ((bp = getnewbuf(slpflag, slptimeo, 0)) == NULL)
1161 			goto loop;
1162 
1163 		if (incore(vp, blkno) != NULL) {
1164 			/* The block has come into memory in the meantime. */
1165 			brelsel(bp, 0);
1166 			goto loop;
1167 		}
1168 
1169 		LIST_INSERT_HEAD(BUFHASH(vp, blkno), bp, b_hash);
1170 		bp->b_blkno = bp->b_lblkno = bp->b_rawblkno = blkno;
1171 		mutex_enter(vp->v_interlock);
1172 		bgetvp(vp, bp);
1173 		mutex_exit(vp->v_interlock);
1174 		preserve = 0;
1175 	}
1176 	mutex_exit(&bufcache_lock);
1177 
1178 	/*
1179 	 * LFS can't track total size of B_LOCKED buffer (locked_queue_bytes)
1180 	 * if we re-size buffers here.
1181 	 */
1182 	if (ISSET(bp->b_flags, B_LOCKED)) {
1183 		KASSERT(bp->b_bufsize >= size);
1184 	} else {
1185 		if (allocbuf(bp, size, preserve)) {
1186 			mutex_enter(&bufcache_lock);
1187 			LIST_REMOVE(bp, b_hash);
1188 			mutex_exit(&bufcache_lock);
1189 			brelse(bp, BC_INVAL);
1190 			return NULL;
1191 		}
1192 	}
1193 	BIO_SETPRIO(bp, BPRIO_DEFAULT);
1194 	return (bp);
1195 }
1196 
1197 /*
1198  * Get an empty, disassociated buffer of given size.
1199  */
1200 buf_t *
1201 geteblk(int size)
1202 {
1203 	buf_t *bp;
1204 	int error;
1205 
1206 	mutex_enter(&bufcache_lock);
1207 	while ((bp = getnewbuf(0, 0, 0)) == NULL)
1208 		;
1209 
1210 	SET(bp->b_cflags, BC_INVAL);
1211 	LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1212 	mutex_exit(&bufcache_lock);
1213 	BIO_SETPRIO(bp, BPRIO_DEFAULT);
1214 	error = allocbuf(bp, size, 0);
1215 	KASSERT(error == 0);
1216 	return (bp);
1217 }
1218 
1219 /*
1220  * Expand or contract the actual memory allocated to a buffer.
1221  *
1222  * If the buffer shrinks, data is lost, so it's up to the
1223  * caller to have written it out *first*; this routine will not
1224  * start a write.  If the buffer grows, it's the callers
1225  * responsibility to fill out the buffer's additional contents.
1226  */
1227 int
1228 allocbuf(buf_t *bp, int size, int preserve)
1229 {
1230 	void *addr;
1231 	vsize_t oldsize, desired_size;
1232 	int oldcount;
1233 	int delta;
1234 
1235 	desired_size = buf_roundsize(size);
1236 	if (desired_size > MAXBSIZE)
1237 		printf("allocbuf: buffer larger than MAXBSIZE requested");
1238 
1239 	oldcount = bp->b_bcount;
1240 
1241 	bp->b_bcount = size;
1242 
1243 	oldsize = bp->b_bufsize;
1244 	if (oldsize == desired_size) {
1245 		/*
1246 		 * Do not short cut the WAPBL resize, as the buffer length
1247 		 * could still have changed and this would corrupt the
1248 		 * tracking of the transaction length.
1249 		 */
1250 		goto out;
1251 	}
1252 
1253 	/*
1254 	 * If we want a buffer of a different size, re-allocate the
1255 	 * buffer's memory; copy old content only if needed.
1256 	 */
1257 	addr = buf_alloc(desired_size);
1258 	if (addr == NULL)
1259 		return ENOMEM;
1260 	if (preserve)
1261 		memcpy(addr, bp->b_data, MIN(oldsize,desired_size));
1262 	if (bp->b_data != NULL)
1263 		buf_mrelease(bp->b_data, oldsize);
1264 	bp->b_data = addr;
1265 	bp->b_bufsize = desired_size;
1266 
1267 	/*
1268 	 * Update overall buffer memory counter (protected by bufcache_lock)
1269 	 */
1270 	delta = (long)desired_size - (long)oldsize;
1271 
1272 	mutex_enter(&bufcache_lock);
1273 	if ((bufmem += delta) > bufmem_hiwater) {
1274 		/*
1275 		 * Need to trim overall memory usage.
1276 		 */
1277 		while (buf_canrelease()) {
1278 			if (curcpu()->ci_schedstate.spc_flags &
1279 			    SPCF_SHOULDYIELD) {
1280 				mutex_exit(&bufcache_lock);
1281 				preempt();
1282 				mutex_enter(&bufcache_lock);
1283 			}
1284 			if (buf_trim() == 0)
1285 				break;
1286 		}
1287 	}
1288 	mutex_exit(&bufcache_lock);
1289 
1290  out:
1291 	if (wapbl_vphaswapbl(bp->b_vp))
1292 		WAPBL_RESIZE_BUF(wapbl_vptomp(bp->b_vp), bp, oldsize, oldcount);
1293 
1294 	return 0;
1295 }
1296 
1297 /*
1298  * Find a buffer which is available for use.
1299  * Select something from a free list.
1300  * Preference is to AGE list, then LRU list.
1301  *
1302  * Called with the buffer queues locked.
1303  * Return buffer locked.
1304  */
1305 buf_t *
1306 getnewbuf(int slpflag, int slptimeo, int from_bufq)
1307 {
1308 	buf_t *bp;
1309 	struct vnode *vp;
1310 
1311  start:
1312 	KASSERT(mutex_owned(&bufcache_lock));
1313 
1314 	/*
1315 	 * Get a new buffer from the pool.
1316 	 */
1317 	if (!from_bufq && buf_lotsfree()) {
1318 		mutex_exit(&bufcache_lock);
1319 		bp = pool_cache_get(buf_cache, PR_NOWAIT);
1320 		if (bp != NULL) {
1321 			memset((char *)bp, 0, sizeof(*bp));
1322 			buf_init(bp);
1323 			SET(bp->b_cflags, BC_BUSY);	/* mark buffer busy */
1324 			mutex_enter(&bufcache_lock);
1325 #if defined(DIAGNOSTIC)
1326 			bp->b_freelistindex = -1;
1327 #endif /* defined(DIAGNOSTIC) */
1328 			return (bp);
1329 		}
1330 		mutex_enter(&bufcache_lock);
1331 	}
1332 
1333 	KASSERT(mutex_owned(&bufcache_lock));
1334 	if ((bp = TAILQ_FIRST(&bufqueues[BQ_AGE].bq_queue)) != NULL ||
1335 	    (bp = TAILQ_FIRST(&bufqueues[BQ_LRU].bq_queue)) != NULL) {
1336 	    	KASSERT(!ISSET(bp->b_cflags, BC_BUSY) || ISSET(bp->b_cflags, BC_VFLUSH));
1337 		bremfree(bp);
1338 
1339 		/* Buffer is no longer on free lists. */
1340 		SET(bp->b_cflags, BC_BUSY);
1341 	} else {
1342 		/*
1343 		 * XXX: !from_bufq should be removed.
1344 		 */
1345 		if (!from_bufq || curlwp != uvm.pagedaemon_lwp) {
1346 			/* wait for a free buffer of any kind */
1347 			if ((slpflag & PCATCH) != 0)
1348 				(void)cv_timedwait_sig(&needbuffer_cv,
1349 				    &bufcache_lock, slptimeo);
1350 			else
1351 				(void)cv_timedwait(&needbuffer_cv,
1352 				    &bufcache_lock, slptimeo);
1353 		}
1354 		return (NULL);
1355 	}
1356 
1357 #ifdef DIAGNOSTIC
1358 	if (bp->b_bufsize <= 0)
1359 		panic("buffer %p: on queue but empty", bp);
1360 #endif
1361 
1362 	if (ISSET(bp->b_cflags, BC_VFLUSH)) {
1363 		/*
1364 		 * This is a delayed write buffer being flushed to disk.  Make
1365 		 * sure it gets aged out of the queue when it's finished, and
1366 		 * leave it off the LRU queue.
1367 		 */
1368 		CLR(bp->b_cflags, BC_VFLUSH);
1369 		SET(bp->b_cflags, BC_AGE);
1370 		goto start;
1371 	}
1372 
1373 	KASSERT(ISSET(bp->b_cflags, BC_BUSY));
1374 	KASSERT(bp->b_refcnt > 0);
1375     	KASSERT(!cv_has_waiters(&bp->b_done));
1376 
1377 	/*
1378 	 * If buffer was a delayed write, start it and return NULL
1379 	 * (since we might sleep while starting the write).
1380 	 */
1381 	if (ISSET(bp->b_oflags, BO_DELWRI)) {
1382 		/*
1383 		 * This buffer has gone through the LRU, so make sure it gets
1384 		 * reused ASAP.
1385 		 */
1386 		SET(bp->b_cflags, BC_AGE);
1387 		mutex_exit(&bufcache_lock);
1388 		bawrite(bp);
1389 		mutex_enter(&bufcache_lock);
1390 		return (NULL);
1391 	}
1392 
1393 	vp = bp->b_vp;
1394 
1395 	/* clear out various other fields */
1396 	bp->b_cflags = BC_BUSY;
1397 	bp->b_oflags = 0;
1398 	bp->b_flags = 0;
1399 	bp->b_dev = NODEV;
1400 	bp->b_blkno = 0;
1401 	bp->b_lblkno = 0;
1402 	bp->b_rawblkno = 0;
1403 	bp->b_iodone = 0;
1404 	bp->b_error = 0;
1405 	bp->b_resid = 0;
1406 	bp->b_bcount = 0;
1407 
1408 	LIST_REMOVE(bp, b_hash);
1409 
1410 	/* Disassociate us from our vnode, if we had one... */
1411 	if (vp != NULL) {
1412 		mutex_enter(vp->v_interlock);
1413 		brelvp(bp);
1414 		mutex_exit(vp->v_interlock);
1415 	}
1416 
1417 	return (bp);
1418 }
1419 
1420 /*
1421  * Attempt to free an aged buffer off the queues.
1422  * Called with queue lock held.
1423  * Returns the amount of buffer memory freed.
1424  */
1425 static int
1426 buf_trim(void)
1427 {
1428 	buf_t *bp;
1429 	long size = 0;
1430 
1431 	KASSERT(mutex_owned(&bufcache_lock));
1432 
1433 	/* Instruct getnewbuf() to get buffers off the queues */
1434 	if ((bp = getnewbuf(PCATCH, 1, 1)) == NULL)
1435 		return 0;
1436 
1437 	KASSERT((bp->b_cflags & BC_WANTED) == 0);
1438 	size = bp->b_bufsize;
1439 	bufmem -= size;
1440 	if (size > 0) {
1441 		buf_mrelease(bp->b_data, size);
1442 		bp->b_bcount = bp->b_bufsize = 0;
1443 	}
1444 	/* brelse() will return the buffer to the global buffer pool */
1445 	brelsel(bp, 0);
1446 	return size;
1447 }
1448 
1449 int
1450 buf_drain(int n)
1451 {
1452 	int size = 0, sz;
1453 
1454 	KASSERT(mutex_owned(&bufcache_lock));
1455 
1456 	while (size < n && bufmem > bufmem_lowater) {
1457 		sz = buf_trim();
1458 		if (sz <= 0)
1459 			break;
1460 		size += sz;
1461 	}
1462 
1463 	return size;
1464 }
1465 
1466 /*
1467  * Wait for operations on the buffer to complete.
1468  * When they do, extract and return the I/O's error value.
1469  */
1470 int
1471 biowait(buf_t *bp)
1472 {
1473 
1474 	KASSERT(ISSET(bp->b_cflags, BC_BUSY));
1475 	KASSERT(bp->b_refcnt > 0);
1476 
1477 	mutex_enter(bp->b_objlock);
1478 	while (!ISSET(bp->b_oflags, BO_DONE | BO_DELWRI))
1479 		cv_wait(&bp->b_done, bp->b_objlock);
1480 	mutex_exit(bp->b_objlock);
1481 
1482 	return bp->b_error;
1483 }
1484 
1485 /*
1486  * Mark I/O complete on a buffer.
1487  *
1488  * If a callback has been requested, e.g. the pageout
1489  * daemon, do so. Otherwise, awaken waiting processes.
1490  *
1491  * [ Leffler, et al., says on p.247:
1492  *	"This routine wakes up the blocked process, frees the buffer
1493  *	for an asynchronous write, or, for a request by the pagedaemon
1494  *	process, invokes a procedure specified in the buffer structure" ]
1495  *
1496  * In real life, the pagedaemon (or other system processes) wants
1497  * to do async stuff to, and doesn't want the buffer brelse()'d.
1498  * (for swap pager, that puts swap buffers on the free lists (!!!),
1499  * for the vn device, that puts allocated buffers on the free lists!)
1500  */
1501 void
1502 biodone(buf_t *bp)
1503 {
1504 	int s;
1505 
1506 	KASSERT(!ISSET(bp->b_oflags, BO_DONE));
1507 
1508 	if (cpu_intr_p()) {
1509 		/* From interrupt mode: defer to a soft interrupt. */
1510 		s = splvm();
1511 		TAILQ_INSERT_TAIL(&curcpu()->ci_data.cpu_biodone, bp, b_actq);
1512 		softint_schedule(biodone_sih);
1513 		splx(s);
1514 	} else {
1515 		/* Process now - the buffer may be freed soon. */
1516 		biodone2(bp);
1517 	}
1518 }
1519 
1520 static void
1521 biodone2(buf_t *bp)
1522 {
1523 	void (*callout)(buf_t *);
1524 
1525 	mutex_enter(bp->b_objlock);
1526 	/* Note that the transfer is done. */
1527 	if (ISSET(bp->b_oflags, BO_DONE))
1528 		panic("biodone2 already");
1529 	CLR(bp->b_flags, B_COWDONE);
1530 	SET(bp->b_oflags, BO_DONE);
1531 	BIO_SETPRIO(bp, BPRIO_DEFAULT);
1532 
1533 	/* Wake up waiting writers. */
1534 	if (!ISSET(bp->b_flags, B_READ))
1535 		vwakeup(bp);
1536 
1537 	if ((callout = bp->b_iodone) != NULL) {
1538 		/* Note callout done, then call out. */
1539 		KASSERT(!cv_has_waiters(&bp->b_done));
1540 		KERNEL_LOCK(1, NULL);		/* XXXSMP */
1541 		bp->b_iodone = NULL;
1542 		mutex_exit(bp->b_objlock);
1543 		(*callout)(bp);
1544 		KERNEL_UNLOCK_ONE(NULL);	/* XXXSMP */
1545 	} else if (ISSET(bp->b_flags, B_ASYNC)) {
1546 		/* If async, release. */
1547 		KASSERT(!cv_has_waiters(&bp->b_done));
1548 		mutex_exit(bp->b_objlock);
1549 		brelse(bp, 0);
1550 	} else {
1551 		/* Otherwise just wake up waiters in biowait(). */
1552 		cv_broadcast(&bp->b_done);
1553 		mutex_exit(bp->b_objlock);
1554 	}
1555 }
1556 
1557 static void
1558 biointr(void *cookie)
1559 {
1560 	struct cpu_info *ci;
1561 	buf_t *bp;
1562 	int s;
1563 
1564 	ci = curcpu();
1565 
1566 	while (!TAILQ_EMPTY(&ci->ci_data.cpu_biodone)) {
1567 		KASSERT(curcpu() == ci);
1568 
1569 		s = splvm();
1570 		bp = TAILQ_FIRST(&ci->ci_data.cpu_biodone);
1571 		TAILQ_REMOVE(&ci->ci_data.cpu_biodone, bp, b_actq);
1572 		splx(s);
1573 
1574 		biodone2(bp);
1575 	}
1576 }
1577 
1578 /*
1579  * Wait for all buffers to complete I/O
1580  * Return the number of "stuck" buffers.
1581  */
1582 int
1583 buf_syncwait(void)
1584 {
1585 	buf_t *bp;
1586 	int iter, nbusy, nbusy_prev = 0, dcount, ihash;
1587 
1588 	dcount = 10000;
1589 	for (iter = 0; iter < 20;) {
1590 		mutex_enter(&bufcache_lock);
1591 		nbusy = 0;
1592 		for (ihash = 0; ihash < bufhash+1; ihash++) {
1593 		    LIST_FOREACH(bp, &bufhashtbl[ihash], b_hash) {
1594 			if ((bp->b_cflags & (BC_BUSY|BC_INVAL)) == BC_BUSY)
1595 				nbusy += ((bp->b_flags & B_READ) == 0);
1596 		    }
1597 		}
1598 		mutex_exit(&bufcache_lock);
1599 
1600 		if (nbusy == 0)
1601 			break;
1602 		if (nbusy_prev == 0)
1603 			nbusy_prev = nbusy;
1604 		printf("%d ", nbusy);
1605 		kpause("bflush", false, MAX(1, hz / 25 * iter), NULL);
1606 		if (nbusy >= nbusy_prev) /* we didn't flush anything */
1607 			iter++;
1608 		else
1609 			nbusy_prev = nbusy;
1610 	}
1611 
1612 	if (nbusy) {
1613 #if defined(DEBUG) || defined(DEBUG_HALT_BUSY)
1614 		printf("giving up\nPrinting vnodes for busy buffers\n");
1615 		for (ihash = 0; ihash < bufhash+1; ihash++) {
1616 		    LIST_FOREACH(bp, &bufhashtbl[ihash], b_hash) {
1617 			if ((bp->b_cflags & (BC_BUSY|BC_INVAL)) == BC_BUSY &&
1618 			    (bp->b_flags & B_READ) == 0)
1619 				vprint(NULL, bp->b_vp);
1620 		    }
1621 		}
1622 #endif
1623 	}
1624 
1625 	return nbusy;
1626 }
1627 
1628 static void
1629 sysctl_fillbuf(buf_t *i, struct buf_sysctl *o)
1630 {
1631 
1632 	o->b_flags = i->b_flags | i->b_cflags | i->b_oflags;
1633 	o->b_error = i->b_error;
1634 	o->b_prio = i->b_prio;
1635 	o->b_dev = i->b_dev;
1636 	o->b_bufsize = i->b_bufsize;
1637 	o->b_bcount = i->b_bcount;
1638 	o->b_resid = i->b_resid;
1639 	o->b_addr = PTRTOUINT64(i->b_data);
1640 	o->b_blkno = i->b_blkno;
1641 	o->b_rawblkno = i->b_rawblkno;
1642 	o->b_iodone = PTRTOUINT64(i->b_iodone);
1643 	o->b_proc = PTRTOUINT64(i->b_proc);
1644 	o->b_vp = PTRTOUINT64(i->b_vp);
1645 	o->b_saveaddr = PTRTOUINT64(i->b_saveaddr);
1646 	o->b_lblkno = i->b_lblkno;
1647 }
1648 
1649 #define KERN_BUFSLOP 20
1650 static int
1651 sysctl_dobuf(SYSCTLFN_ARGS)
1652 {
1653 	buf_t *bp;
1654 	struct buf_sysctl bs;
1655 	struct bqueue *bq;
1656 	char *dp;
1657 	u_int i, op, arg;
1658 	size_t len, needed, elem_size, out_size;
1659 	int error, elem_count, retries;
1660 
1661 	if (namelen == 1 && name[0] == CTL_QUERY)
1662 		return (sysctl_query(SYSCTLFN_CALL(rnode)));
1663 
1664 	if (namelen != 4)
1665 		return (EINVAL);
1666 
1667 	retries = 100;
1668  retry:
1669 	dp = oldp;
1670 	len = (oldp != NULL) ? *oldlenp : 0;
1671 	op = name[0];
1672 	arg = name[1];
1673 	elem_size = name[2];
1674 	elem_count = name[3];
1675 	out_size = MIN(sizeof(bs), elem_size);
1676 
1677 	/*
1678 	 * at the moment, these are just "placeholders" to make the
1679 	 * API for retrieving kern.buf data more extensible in the
1680 	 * future.
1681 	 *
1682 	 * XXX kern.buf currently has "netbsd32" issues.  hopefully
1683 	 * these will be resolved at a later point.
1684 	 */
1685 	if (op != KERN_BUF_ALL || arg != KERN_BUF_ALL ||
1686 	    elem_size < 1 || elem_count < 0)
1687 		return (EINVAL);
1688 
1689 	error = 0;
1690 	needed = 0;
1691 	sysctl_unlock();
1692 	mutex_enter(&bufcache_lock);
1693 	for (i = 0; i < BQUEUES; i++) {
1694 		bq = &bufqueues[i];
1695 		TAILQ_FOREACH(bp, &bq->bq_queue, b_freelist) {
1696 			bq->bq_marker = bp;
1697 			if (len >= elem_size && elem_count > 0) {
1698 				sysctl_fillbuf(bp, &bs);
1699 				mutex_exit(&bufcache_lock);
1700 				error = copyout(&bs, dp, out_size);
1701 				mutex_enter(&bufcache_lock);
1702 				if (error)
1703 					break;
1704 				if (bq->bq_marker != bp) {
1705 					/*
1706 					 * This sysctl node is only for
1707 					 * statistics.  Retry; if the
1708 					 * queue keeps changing, then
1709 					 * bail out.
1710 					 */
1711 					if (retries-- == 0) {
1712 						error = EAGAIN;
1713 						break;
1714 					}
1715 					mutex_exit(&bufcache_lock);
1716 					goto retry;
1717 				}
1718 				dp += elem_size;
1719 				len -= elem_size;
1720 			}
1721 			needed += elem_size;
1722 			if (elem_count > 0 && elem_count != INT_MAX)
1723 				elem_count--;
1724 		}
1725 		if (error != 0)
1726 			break;
1727 	}
1728 	mutex_exit(&bufcache_lock);
1729 	sysctl_relock();
1730 
1731 	*oldlenp = needed;
1732 	if (oldp == NULL)
1733 		*oldlenp += KERN_BUFSLOP * sizeof(buf_t);
1734 
1735 	return (error);
1736 }
1737 
1738 static int
1739 sysctl_bufvm_update(SYSCTLFN_ARGS)
1740 {
1741 	int t, error, rv;
1742 	struct sysctlnode node;
1743 
1744 	node = *rnode;
1745 	node.sysctl_data = &t;
1746 	t = *(int *)rnode->sysctl_data;
1747 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
1748 	if (error || newp == NULL)
1749 		return (error);
1750 
1751 	if (t < 0)
1752 		return EINVAL;
1753 	if (rnode->sysctl_data == &bufcache) {
1754 		if (t > 100)
1755 			return (EINVAL);
1756 		bufcache = t;
1757 		buf_setwm();
1758 	} else if (rnode->sysctl_data == &bufmem_lowater) {
1759 		if (bufmem_hiwater - t < 16)
1760 			return (EINVAL);
1761 		bufmem_lowater = t;
1762 	} else if (rnode->sysctl_data == &bufmem_hiwater) {
1763 		if (t - bufmem_lowater < 16)
1764 			return (EINVAL);
1765 		bufmem_hiwater = t;
1766 	} else
1767 		return (EINVAL);
1768 
1769 	/* Drain until below new high water mark */
1770 	sysctl_unlock();
1771 	mutex_enter(&bufcache_lock);
1772 	while ((t = bufmem - bufmem_hiwater) >= 0) {
1773 		rv = buf_drain(t / (2 * 1024));
1774 		if (rv <= 0)
1775 			break;
1776 	}
1777 	mutex_exit(&bufcache_lock);
1778 	sysctl_relock();
1779 
1780 	return 0;
1781 }
1782 
1783 static struct sysctllog *vfsbio_sysctllog;
1784 
1785 static void
1786 sysctl_kern_buf_setup(void)
1787 {
1788 
1789 	sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1790 		       CTLFLAG_PERMANENT,
1791 		       CTLTYPE_NODE, "kern", NULL,
1792 		       NULL, 0, NULL, 0,
1793 		       CTL_KERN, CTL_EOL);
1794 	sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1795 		       CTLFLAG_PERMANENT,
1796 		       CTLTYPE_NODE, "buf",
1797 		       SYSCTL_DESCR("Kernel buffer cache information"),
1798 		       sysctl_dobuf, 0, NULL, 0,
1799 		       CTL_KERN, KERN_BUF, CTL_EOL);
1800 }
1801 
1802 static void
1803 sysctl_vm_buf_setup(void)
1804 {
1805 
1806 	sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1807 		       CTLFLAG_PERMANENT,
1808 		       CTLTYPE_NODE, "vm", NULL,
1809 		       NULL, 0, NULL, 0,
1810 		       CTL_VM, CTL_EOL);
1811 	sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1812 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1813 		       CTLTYPE_INT, "bufcache",
1814 		       SYSCTL_DESCR("Percentage of physical memory to use for "
1815 				    "buffer cache"),
1816 		       sysctl_bufvm_update, 0, &bufcache, 0,
1817 		       CTL_VM, CTL_CREATE, CTL_EOL);
1818 	sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1819 		       CTLFLAG_PERMANENT|CTLFLAG_READONLY,
1820 		       CTLTYPE_INT, "bufmem",
1821 		       SYSCTL_DESCR("Amount of kernel memory used by buffer "
1822 				    "cache"),
1823 		       NULL, 0, &bufmem, 0,
1824 		       CTL_VM, CTL_CREATE, CTL_EOL);
1825 	sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1826 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1827 		       CTLTYPE_INT, "bufmem_lowater",
1828 		       SYSCTL_DESCR("Minimum amount of kernel memory to "
1829 				    "reserve for buffer cache"),
1830 		       sysctl_bufvm_update, 0, &bufmem_lowater, 0,
1831 		       CTL_VM, CTL_CREATE, CTL_EOL);
1832 	sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL,
1833 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1834 		       CTLTYPE_INT, "bufmem_hiwater",
1835 		       SYSCTL_DESCR("Maximum amount of kernel memory to use "
1836 				    "for buffer cache"),
1837 		       sysctl_bufvm_update, 0, &bufmem_hiwater, 0,
1838 		       CTL_VM, CTL_CREATE, CTL_EOL);
1839 }
1840 
1841 #ifdef DEBUG
1842 /*
1843  * Print out statistics on the current allocation of the buffer pool.
1844  * Can be enabled to print out on every ``sync'' by setting "syncprt"
1845  * in vfs_syscalls.c using sysctl.
1846  */
1847 void
1848 vfs_bufstats(void)
1849 {
1850 	int i, j, count;
1851 	buf_t *bp;
1852 	struct bqueue *dp;
1853 	int counts[(MAXBSIZE / PAGE_SIZE) + 1];
1854 	static const char *bname[BQUEUES] = { "LOCKED", "LRU", "AGE" };
1855 
1856 	for (dp = bufqueues, i = 0; dp < &bufqueues[BQUEUES]; dp++, i++) {
1857 		count = 0;
1858 		for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++)
1859 			counts[j] = 0;
1860 		TAILQ_FOREACH(bp, &dp->bq_queue, b_freelist) {
1861 			counts[bp->b_bufsize/PAGE_SIZE]++;
1862 			count++;
1863 		}
1864 		printf("%s: total-%d", bname[i], count);
1865 		for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++)
1866 			if (counts[j] != 0)
1867 				printf(", %d-%d", j * PAGE_SIZE, counts[j]);
1868 		printf("\n");
1869 	}
1870 }
1871 #endif /* DEBUG */
1872 
1873 /* ------------------------------ */
1874 
1875 buf_t *
1876 getiobuf(struct vnode *vp, bool waitok)
1877 {
1878 	buf_t *bp;
1879 
1880 	bp = pool_cache_get(bufio_cache, (waitok ? PR_WAITOK : PR_NOWAIT));
1881 	if (bp == NULL)
1882 		return bp;
1883 
1884 	buf_init(bp);
1885 
1886 	if ((bp->b_vp = vp) == NULL)
1887 		bp->b_objlock = &buffer_lock;
1888 	else
1889 		bp->b_objlock = vp->v_interlock;
1890 
1891 	return bp;
1892 }
1893 
1894 void
1895 putiobuf(buf_t *bp)
1896 {
1897 
1898 	buf_destroy(bp);
1899 	pool_cache_put(bufio_cache, bp);
1900 }
1901 
1902 /*
1903  * nestiobuf_iodone: b_iodone callback for nested buffers.
1904  */
1905 
1906 void
1907 nestiobuf_iodone(buf_t *bp)
1908 {
1909 	buf_t *mbp = bp->b_private;
1910 	int error;
1911 	int donebytes;
1912 
1913 	KASSERT(bp->b_bcount <= bp->b_bufsize);
1914 	KASSERT(mbp != bp);
1915 
1916 	error = bp->b_error;
1917 	if (bp->b_error == 0 &&
1918 	    (bp->b_bcount < bp->b_bufsize || bp->b_resid > 0)) {
1919 		/*
1920 		 * Not all got transfered, raise an error. We have no way to
1921 		 * propagate these conditions to mbp.
1922 		 */
1923 		error = EIO;
1924 	}
1925 
1926 	donebytes = bp->b_bufsize;
1927 
1928 	putiobuf(bp);
1929 	nestiobuf_done(mbp, donebytes, error);
1930 }
1931 
1932 /*
1933  * nestiobuf_setup: setup a "nested" buffer.
1934  *
1935  * => 'mbp' is a "master" buffer which is being divided into sub pieces.
1936  * => 'bp' should be a buffer allocated by getiobuf.
1937  * => 'offset' is a byte offset in the master buffer.
1938  * => 'size' is a size in bytes of this nested buffer.
1939  */
1940 
1941 void
1942 nestiobuf_setup(buf_t *mbp, buf_t *bp, int offset, size_t size)
1943 {
1944 	const int b_read = mbp->b_flags & B_READ;
1945 	struct vnode *vp = mbp->b_vp;
1946 
1947 	KASSERT(mbp->b_bcount >= offset + size);
1948 	bp->b_vp = vp;
1949 	bp->b_dev = mbp->b_dev;
1950 	bp->b_objlock = mbp->b_objlock;
1951 	bp->b_cflags = BC_BUSY;
1952 	bp->b_flags = B_ASYNC | b_read;
1953 	bp->b_iodone = nestiobuf_iodone;
1954 	bp->b_data = (char *)mbp->b_data + offset;
1955 	bp->b_resid = bp->b_bcount = size;
1956 	bp->b_bufsize = bp->b_bcount;
1957 	bp->b_private = mbp;
1958 	BIO_COPYPRIO(bp, mbp);
1959 	if (!b_read && vp != NULL) {
1960 		mutex_enter(vp->v_interlock);
1961 		vp->v_numoutput++;
1962 		mutex_exit(vp->v_interlock);
1963 	}
1964 }
1965 
1966 /*
1967  * nestiobuf_done: propagate completion to the master buffer.
1968  *
1969  * => 'donebytes' specifies how many bytes in the 'mbp' is completed.
1970  * => 'error' is an errno(2) that 'donebytes' has been completed with.
1971  */
1972 
1973 void
1974 nestiobuf_done(buf_t *mbp, int donebytes, int error)
1975 {
1976 
1977 	if (donebytes == 0) {
1978 		return;
1979 	}
1980 	mutex_enter(mbp->b_objlock);
1981 	KASSERT(mbp->b_resid >= donebytes);
1982 	mbp->b_resid -= donebytes;
1983 	if (error)
1984 		mbp->b_error = error;
1985 	if (mbp->b_resid == 0) {
1986 		if (mbp->b_error)
1987 			mbp->b_resid = mbp->b_bcount;
1988 		mutex_exit(mbp->b_objlock);
1989 		biodone(mbp);
1990 	} else
1991 		mutex_exit(mbp->b_objlock);
1992 }
1993 
1994 void
1995 buf_init(buf_t *bp)
1996 {
1997 
1998 	cv_init(&bp->b_busy, "biolock");
1999 	cv_init(&bp->b_done, "biowait");
2000 	bp->b_dev = NODEV;
2001 	bp->b_error = 0;
2002 	bp->b_flags = 0;
2003 	bp->b_cflags = 0;
2004 	bp->b_oflags = 0;
2005 	bp->b_objlock = &buffer_lock;
2006 	bp->b_iodone = NULL;
2007 	bp->b_refcnt = 1;
2008 	bp->b_dev = NODEV;
2009 	bp->b_vnbufs.le_next = NOLIST;
2010 	BIO_SETPRIO(bp, BPRIO_DEFAULT);
2011 }
2012 
2013 void
2014 buf_destroy(buf_t *bp)
2015 {
2016 
2017 	cv_destroy(&bp->b_done);
2018 	cv_destroy(&bp->b_busy);
2019 }
2020 
2021 int
2022 bbusy(buf_t *bp, bool intr, int timo, kmutex_t *interlock)
2023 {
2024 	int error;
2025 
2026 	KASSERT(mutex_owned(&bufcache_lock));
2027 
2028 	if ((bp->b_cflags & BC_BUSY) != 0) {
2029 		if (curlwp == uvm.pagedaemon_lwp)
2030 			return EDEADLK;
2031 		bp->b_cflags |= BC_WANTED;
2032 		bref(bp);
2033 		if (interlock != NULL)
2034 			mutex_exit(interlock);
2035 		if (intr) {
2036 			error = cv_timedwait_sig(&bp->b_busy, &bufcache_lock,
2037 			    timo);
2038 		} else {
2039 			error = cv_timedwait(&bp->b_busy, &bufcache_lock,
2040 			    timo);
2041 		}
2042 		brele(bp);
2043 		if (interlock != NULL)
2044 			mutex_enter(interlock);
2045 		if (error != 0)
2046 			return error;
2047 		return EPASSTHROUGH;
2048 	}
2049 	bp->b_cflags |= BC_BUSY;
2050 
2051 	return 0;
2052 }
2053