xref: /netbsd-src/sys/kern/sys_pipe.c (revision a536ee5124e62c9a0051a252f7833dc8f50f44c9)
1 /*	$NetBSD: sys_pipe.c,v 1.136 2012/05/16 09:41:11 martin Exp $	*/
2 
3 /*-
4  * Copyright (c) 2003, 2007, 2008, 2009 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Paul Kranenburg, and by Andrew Doran.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 /*
33  * Copyright (c) 1996 John S. Dyson
34  * All rights reserved.
35  *
36  * Redistribution and use in source and binary forms, with or without
37  * modification, are permitted provided that the following conditions
38  * are met:
39  * 1. Redistributions of source code must retain the above copyright
40  *    notice immediately at the beginning of the file, without modification,
41  *    this list of conditions, and the following disclaimer.
42  * 2. Redistributions in binary form must reproduce the above copyright
43  *    notice, this list of conditions and the following disclaimer in the
44  *    documentation and/or other materials provided with the distribution.
45  * 3. Absolutely no warranty of function or purpose is made by the author
46  *    John S. Dyson.
47  * 4. Modifications may be freely made to this file if the above conditions
48  *    are met.
49  */
50 
51 /*
52  * This file contains a high-performance replacement for the socket-based
53  * pipes scheme originally used.  It does not support all features of
54  * sockets, but does do everything that pipes normally do.
55  *
56  * This code has two modes of operation, a small write mode and a large
57  * write mode.  The small write mode acts like conventional pipes with
58  * a kernel buffer.  If the buffer is less than PIPE_MINDIRECT, then the
59  * "normal" pipe buffering is done.  If the buffer is between PIPE_MINDIRECT
60  * and PIPE_SIZE in size it is mapped read-only into the kernel address space
61  * using the UVM page loan facility from where the receiving process can copy
62  * the data directly from the pages in the sending process.
63  *
64  * The constant PIPE_MINDIRECT is chosen to make sure that buffering will
65  * happen for small transfers so that the system will not spend all of
66  * its time context switching.  PIPE_SIZE is constrained by the
67  * amount of kernel virtual memory.
68  */
69 
70 #include <sys/cdefs.h>
71 __KERNEL_RCSID(0, "$NetBSD: sys_pipe.c,v 1.136 2012/05/16 09:41:11 martin Exp $");
72 
73 #include <sys/param.h>
74 #include <sys/systm.h>
75 #include <sys/proc.h>
76 #include <sys/fcntl.h>
77 #include <sys/file.h>
78 #include <sys/filedesc.h>
79 #include <sys/filio.h>
80 #include <sys/kernel.h>
81 #include <sys/ttycom.h>
82 #include <sys/stat.h>
83 #include <sys/poll.h>
84 #include <sys/signalvar.h>
85 #include <sys/vnode.h>
86 #include <sys/uio.h>
87 #include <sys/select.h>
88 #include <sys/mount.h>
89 #include <sys/syscallargs.h>
90 #include <sys/sysctl.h>
91 #include <sys/kauth.h>
92 #include <sys/atomic.h>
93 #include <sys/pipe.h>
94 
95 #include <uvm/uvm_extern.h>
96 
97 /*
98  * Use this to disable direct I/O and decrease the code size:
99  * #define PIPE_NODIRECT
100  */
101 
102 /* XXX Disabled for now; rare hangs switching between direct/buffered */
103 #define PIPE_NODIRECT
104 
105 static int	pipe_read(file_t *, off_t *, struct uio *, kauth_cred_t, int);
106 static int	pipe_write(file_t *, off_t *, struct uio *, kauth_cred_t, int);
107 static int	pipe_close(file_t *);
108 static int	pipe_poll(file_t *, int);
109 static int	pipe_kqfilter(file_t *, struct knote *);
110 static int	pipe_stat(file_t *, struct stat *);
111 static int	pipe_ioctl(file_t *, u_long, void *);
112 static void	pipe_restart(file_t *);
113 
114 static const struct fileops pipeops = {
115 	.fo_read = pipe_read,
116 	.fo_write = pipe_write,
117 	.fo_ioctl = pipe_ioctl,
118 	.fo_fcntl = fnullop_fcntl,
119 	.fo_poll = pipe_poll,
120 	.fo_stat = pipe_stat,
121 	.fo_close = pipe_close,
122 	.fo_kqfilter = pipe_kqfilter,
123 	.fo_restart = pipe_restart,
124 };
125 
126 /*
127  * Default pipe buffer size(s), this can be kind-of large now because pipe
128  * space is pageable.  The pipe code will try to maintain locality of
129  * reference for performance reasons, so small amounts of outstanding I/O
130  * will not wipe the cache.
131  */
132 #define	MINPIPESIZE	(PIPE_SIZE / 3)
133 #define	MAXPIPESIZE	(2 * PIPE_SIZE / 3)
134 
135 /*
136  * Maximum amount of kva for pipes -- this is kind-of a soft limit, but
137  * is there so that on large systems, we don't exhaust it.
138  */
139 #define	MAXPIPEKVA	(8 * 1024 * 1024)
140 static u_int	maxpipekva = MAXPIPEKVA;
141 
142 /*
143  * Limit for direct transfers, we cannot, of course limit
144  * the amount of kva for pipes in general though.
145  */
146 #define	LIMITPIPEKVA	(16 * 1024 * 1024)
147 static u_int	limitpipekva = LIMITPIPEKVA;
148 
149 /*
150  * Limit the number of "big" pipes
151  */
152 #define	LIMITBIGPIPES	32
153 static u_int	maxbigpipes = LIMITBIGPIPES;
154 static u_int	nbigpipe = 0;
155 
156 /*
157  * Amount of KVA consumed by pipe buffers.
158  */
159 static u_int	amountpipekva = 0;
160 
161 static void	pipeclose(struct pipe *);
162 static void	pipe_free_kmem(struct pipe *);
163 static int	pipe_create(struct pipe **, pool_cache_t);
164 static int	pipelock(struct pipe *, int);
165 static inline void pipeunlock(struct pipe *);
166 static void	pipeselwakeup(struct pipe *, struct pipe *, int);
167 #ifndef PIPE_NODIRECT
168 static int	pipe_direct_write(file_t *, struct pipe *, struct uio *);
169 #endif
170 static int	pipespace(struct pipe *, int);
171 static int	pipe_ctor(void *, void *, int);
172 static void	pipe_dtor(void *, void *);
173 
174 #ifndef PIPE_NODIRECT
175 static int	pipe_loan_alloc(struct pipe *, int);
176 static void	pipe_loan_free(struct pipe *);
177 #endif /* PIPE_NODIRECT */
178 
179 static pool_cache_t	pipe_wr_cache;
180 static pool_cache_t	pipe_rd_cache;
181 
182 void
183 pipe_init(void)
184 {
185 
186 	/* Writer side is not automatically allocated KVA. */
187 	pipe_wr_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "pipewr",
188 	    NULL, IPL_NONE, pipe_ctor, pipe_dtor, NULL);
189 	KASSERT(pipe_wr_cache != NULL);
190 
191 	/* Reader side gets preallocated KVA. */
192 	pipe_rd_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "piperd",
193 	    NULL, IPL_NONE, pipe_ctor, pipe_dtor, (void *)1);
194 	KASSERT(pipe_rd_cache != NULL);
195 }
196 
197 static int
198 pipe_ctor(void *arg, void *obj, int flags)
199 {
200 	struct pipe *pipe;
201 	vaddr_t va;
202 
203 	pipe = obj;
204 
205 	memset(pipe, 0, sizeof(struct pipe));
206 	if (arg != NULL) {
207 		/* Preallocate space. */
208 		va = uvm_km_alloc(kernel_map, PIPE_SIZE, 0,
209 		    UVM_KMF_PAGEABLE | UVM_KMF_WAITVA);
210 		KASSERT(va != 0);
211 		pipe->pipe_kmem = va;
212 		atomic_add_int(&amountpipekva, PIPE_SIZE);
213 	}
214 	cv_init(&pipe->pipe_rcv, "pipe_rd");
215 	cv_init(&pipe->pipe_wcv, "pipe_wr");
216 	cv_init(&pipe->pipe_draincv, "pipe_drn");
217 	cv_init(&pipe->pipe_lkcv, "pipe_lk");
218 	selinit(&pipe->pipe_sel);
219 	pipe->pipe_state = PIPE_SIGNALR;
220 
221 	return 0;
222 }
223 
224 static void
225 pipe_dtor(void *arg, void *obj)
226 {
227 	struct pipe *pipe;
228 
229 	pipe = obj;
230 
231 	cv_destroy(&pipe->pipe_rcv);
232 	cv_destroy(&pipe->pipe_wcv);
233 	cv_destroy(&pipe->pipe_draincv);
234 	cv_destroy(&pipe->pipe_lkcv);
235 	seldestroy(&pipe->pipe_sel);
236 	if (pipe->pipe_kmem != 0) {
237 		uvm_km_free(kernel_map, pipe->pipe_kmem, PIPE_SIZE,
238 		    UVM_KMF_PAGEABLE);
239 		atomic_add_int(&amountpipekva, -PIPE_SIZE);
240 	}
241 }
242 
243 /*
244  * The pipe system call for the DTYPE_PIPE type of pipes
245  */
246 int
247 pipe1(struct lwp *l, register_t *retval, int flags)
248 {
249 	struct pipe *rpipe, *wpipe;
250 	file_t *rf, *wf;
251 	int fd, error;
252 	proc_t *p;
253 
254 	if (flags & ~(O_CLOEXEC|O_NONBLOCK|O_NOSIGPIPE))
255 		return EINVAL;
256 	p = curproc;
257 	rpipe = wpipe = NULL;
258 	if ((error = pipe_create(&rpipe, pipe_rd_cache)) ||
259 	    (error = pipe_create(&wpipe, pipe_wr_cache))) {
260 		goto free2;
261 	}
262 	rpipe->pipe_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
263 	wpipe->pipe_lock = rpipe->pipe_lock;
264 	mutex_obj_hold(wpipe->pipe_lock);
265 
266 	error = fd_allocfile(&rf, &fd);
267 	if (error)
268 		goto free2;
269 	retval[0] = fd;
270 
271 	error = fd_allocfile(&wf, &fd);
272 	if (error)
273 		goto free3;
274 	retval[1] = fd;
275 
276 	rf->f_flag = FREAD | flags;
277 	rf->f_type = DTYPE_PIPE;
278 	rf->f_data = (void *)rpipe;
279 	rf->f_ops = &pipeops;
280 	fd_set_exclose(l, (int)retval[0], (flags & O_CLOEXEC) != 0);
281 
282 	wf->f_flag = FWRITE | flags;
283 	wf->f_type = DTYPE_PIPE;
284 	wf->f_data = (void *)wpipe;
285 	wf->f_ops = &pipeops;
286 	fd_set_exclose(l, (int)retval[1], (flags & O_CLOEXEC) != 0);
287 
288 	rpipe->pipe_peer = wpipe;
289 	wpipe->pipe_peer = rpipe;
290 
291 	fd_affix(p, rf, (int)retval[0]);
292 	fd_affix(p, wf, (int)retval[1]);
293 	return (0);
294 free3:
295 	fd_abort(p, rf, (int)retval[0]);
296 free2:
297 	pipeclose(wpipe);
298 	pipeclose(rpipe);
299 
300 	return (error);
301 }
302 
303 /*
304  * Allocate kva for pipe circular buffer, the space is pageable
305  * This routine will 'realloc' the size of a pipe safely, if it fails
306  * it will retain the old buffer.
307  * If it fails it will return ENOMEM.
308  */
309 static int
310 pipespace(struct pipe *pipe, int size)
311 {
312 	void *buffer;
313 
314 	/*
315 	 * Allocate pageable virtual address space.  Physical memory is
316 	 * allocated on demand.
317 	 */
318 	if (size == PIPE_SIZE && pipe->pipe_kmem != 0) {
319 		buffer = (void *)pipe->pipe_kmem;
320 	} else {
321 		buffer = (void *)uvm_km_alloc(kernel_map, round_page(size),
322 		    0, UVM_KMF_PAGEABLE);
323 		if (buffer == NULL)
324 			return (ENOMEM);
325 		atomic_add_int(&amountpipekva, size);
326 	}
327 
328 	/* free old resources if we're resizing */
329 	pipe_free_kmem(pipe);
330 	pipe->pipe_buffer.buffer = buffer;
331 	pipe->pipe_buffer.size = size;
332 	pipe->pipe_buffer.in = 0;
333 	pipe->pipe_buffer.out = 0;
334 	pipe->pipe_buffer.cnt = 0;
335 	return (0);
336 }
337 
338 /*
339  * Initialize and allocate VM and memory for pipe.
340  */
341 static int
342 pipe_create(struct pipe **pipep, pool_cache_t cache)
343 {
344 	struct pipe *pipe;
345 	int error;
346 
347 	pipe = pool_cache_get(cache, PR_WAITOK);
348 	KASSERT(pipe != NULL);
349 	*pipep = pipe;
350 	error = 0;
351 	getnanotime(&pipe->pipe_btime);
352 	pipe->pipe_atime = pipe->pipe_mtime = pipe->pipe_btime;
353 	pipe->pipe_lock = NULL;
354 	if (cache == pipe_rd_cache) {
355 		error = pipespace(pipe, PIPE_SIZE);
356 	} else {
357 		pipe->pipe_buffer.buffer = NULL;
358 		pipe->pipe_buffer.size = 0;
359 		pipe->pipe_buffer.in = 0;
360 		pipe->pipe_buffer.out = 0;
361 		pipe->pipe_buffer.cnt = 0;
362 	}
363 	return error;
364 }
365 
366 /*
367  * Lock a pipe for I/O, blocking other access
368  * Called with pipe spin lock held.
369  */
370 static int
371 pipelock(struct pipe *pipe, int catch)
372 {
373 	int error;
374 
375 	KASSERT(mutex_owned(pipe->pipe_lock));
376 
377 	while (pipe->pipe_state & PIPE_LOCKFL) {
378 		pipe->pipe_state |= PIPE_LWANT;
379 		if (catch) {
380 			error = cv_wait_sig(&pipe->pipe_lkcv, pipe->pipe_lock);
381 			if (error != 0)
382 				return error;
383 		} else
384 			cv_wait(&pipe->pipe_lkcv, pipe->pipe_lock);
385 	}
386 
387 	pipe->pipe_state |= PIPE_LOCKFL;
388 
389 	return 0;
390 }
391 
392 /*
393  * unlock a pipe I/O lock
394  */
395 static inline void
396 pipeunlock(struct pipe *pipe)
397 {
398 
399 	KASSERT(pipe->pipe_state & PIPE_LOCKFL);
400 
401 	pipe->pipe_state &= ~PIPE_LOCKFL;
402 	if (pipe->pipe_state & PIPE_LWANT) {
403 		pipe->pipe_state &= ~PIPE_LWANT;
404 		cv_broadcast(&pipe->pipe_lkcv);
405 	}
406 }
407 
408 /*
409  * Select/poll wakup. This also sends SIGIO to peer connected to
410  * 'sigpipe' side of pipe.
411  */
412 static void
413 pipeselwakeup(struct pipe *selp, struct pipe *sigp, int code)
414 {
415 	int band;
416 
417 	switch (code) {
418 	case POLL_IN:
419 		band = POLLIN|POLLRDNORM;
420 		break;
421 	case POLL_OUT:
422 		band = POLLOUT|POLLWRNORM;
423 		break;
424 	case POLL_HUP:
425 		band = POLLHUP;
426 		break;
427 	case POLL_ERR:
428 		band = POLLERR;
429 		break;
430 	default:
431 		band = 0;
432 #ifdef DIAGNOSTIC
433 		printf("bad siginfo code %d in pipe notification.\n", code);
434 #endif
435 		break;
436 	}
437 
438 	selnotify(&selp->pipe_sel, band, NOTE_SUBMIT);
439 
440 	if (sigp == NULL || (sigp->pipe_state & PIPE_ASYNC) == 0)
441 		return;
442 
443 	fownsignal(sigp->pipe_pgid, SIGIO, code, band, selp);
444 }
445 
446 static int
447 pipe_read(file_t *fp, off_t *offset, struct uio *uio, kauth_cred_t cred,
448     int flags)
449 {
450 	struct pipe *rpipe = (struct pipe *) fp->f_data;
451 	struct pipebuf *bp = &rpipe->pipe_buffer;
452 	kmutex_t *lock = rpipe->pipe_lock;
453 	int error;
454 	size_t nread = 0;
455 	size_t size;
456 	size_t ocnt;
457 	unsigned int wakeup_state = 0;
458 
459 	mutex_enter(lock);
460 	++rpipe->pipe_busy;
461 	ocnt = bp->cnt;
462 
463 again:
464 	error = pipelock(rpipe, 1);
465 	if (error)
466 		goto unlocked_error;
467 
468 	while (uio->uio_resid) {
469 		/*
470 		 * Normal pipe buffer receive.
471 		 */
472 		if (bp->cnt > 0) {
473 			size = bp->size - bp->out;
474 			if (size > bp->cnt)
475 				size = bp->cnt;
476 			if (size > uio->uio_resid)
477 				size = uio->uio_resid;
478 
479 			mutex_exit(lock);
480 			error = uiomove((char *)bp->buffer + bp->out, size, uio);
481 			mutex_enter(lock);
482 			if (error)
483 				break;
484 
485 			bp->out += size;
486 			if (bp->out >= bp->size)
487 				bp->out = 0;
488 
489 			bp->cnt -= size;
490 
491 			/*
492 			 * If there is no more to read in the pipe, reset
493 			 * its pointers to the beginning.  This improves
494 			 * cache hit stats.
495 			 */
496 			if (bp->cnt == 0) {
497 				bp->in = 0;
498 				bp->out = 0;
499 			}
500 			nread += size;
501 			continue;
502 		}
503 
504 #ifndef PIPE_NODIRECT
505 		if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) {
506 			/*
507 			 * Direct copy, bypassing a kernel buffer.
508 			 */
509 			void *va;
510 			u_int gen;
511 
512 			KASSERT(rpipe->pipe_state & PIPE_DIRECTW);
513 
514 			size = rpipe->pipe_map.cnt;
515 			if (size > uio->uio_resid)
516 				size = uio->uio_resid;
517 
518 			va = (char *)rpipe->pipe_map.kva + rpipe->pipe_map.pos;
519 			gen = rpipe->pipe_map.egen;
520 			mutex_exit(lock);
521 
522 			/*
523 			 * Consume emap and read the data from loaned pages.
524 			 */
525 			uvm_emap_consume(gen);
526 			error = uiomove(va, size, uio);
527 
528 			mutex_enter(lock);
529 			if (error)
530 				break;
531 			nread += size;
532 			rpipe->pipe_map.pos += size;
533 			rpipe->pipe_map.cnt -= size;
534 			if (rpipe->pipe_map.cnt == 0) {
535 				rpipe->pipe_state &= ~PIPE_DIRECTR;
536 				cv_broadcast(&rpipe->pipe_wcv);
537 			}
538 			continue;
539 		}
540 #endif
541 		/*
542 		 * Break if some data was read.
543 		 */
544 		if (nread > 0)
545 			break;
546 
547 		/*
548 		 * Detect EOF condition.
549 		 * Read returns 0 on EOF, no need to set error.
550 		 */
551 		if (rpipe->pipe_state & PIPE_EOF)
552 			break;
553 
554 		/*
555 		 * Don't block on non-blocking I/O.
556 		 */
557 		if (fp->f_flag & FNONBLOCK) {
558 			error = EAGAIN;
559 			break;
560 		}
561 
562 		/*
563 		 * Unlock the pipe buffer for our remaining processing.
564 		 * We will either break out with an error or we will
565 		 * sleep and relock to loop.
566 		 */
567 		pipeunlock(rpipe);
568 
569 		/*
570 		 * Re-check to see if more direct writes are pending.
571 		 */
572 		if ((rpipe->pipe_state & PIPE_DIRECTR) != 0)
573 			goto again;
574 
575 #if 1   /* XXX (dsl) I'm sure these aren't needed here ... */
576 		/*
577 		 * We want to read more, wake up select/poll.
578 		 */
579 		pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT);
580 
581 		/*
582 		 * If the "write-side" is blocked, wake it up now.
583 		 */
584 		cv_broadcast(&rpipe->pipe_wcv);
585 #endif
586 
587 		if (wakeup_state & PIPE_RESTART) {
588 			error = ERESTART;
589 			goto unlocked_error;
590 		}
591 
592 		/* Now wait until the pipe is filled */
593 		error = cv_wait_sig(&rpipe->pipe_rcv, lock);
594 		if (error != 0)
595 			goto unlocked_error;
596 		wakeup_state = rpipe->pipe_state;
597 		goto again;
598 	}
599 
600 	if (error == 0)
601 		getnanotime(&rpipe->pipe_atime);
602 	pipeunlock(rpipe);
603 
604 unlocked_error:
605 	--rpipe->pipe_busy;
606 	if (rpipe->pipe_busy == 0) {
607 		rpipe->pipe_state &= ~PIPE_RESTART;
608 		cv_broadcast(&rpipe->pipe_draincv);
609 	}
610 	if (bp->cnt < MINPIPESIZE) {
611 		cv_broadcast(&rpipe->pipe_wcv);
612 	}
613 
614 	/*
615 	 * If anything was read off the buffer, signal to the writer it's
616 	 * possible to write more data. Also send signal if we are here for the
617 	 * first time after last write.
618 	 */
619 	if ((bp->size - bp->cnt) >= PIPE_BUF
620 	    && (ocnt != bp->cnt || (rpipe->pipe_state & PIPE_SIGNALR))) {
621 		pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT);
622 		rpipe->pipe_state &= ~PIPE_SIGNALR;
623 	}
624 
625 	mutex_exit(lock);
626 	return (error);
627 }
628 
629 #ifndef PIPE_NODIRECT
630 /*
631  * Allocate structure for loan transfer.
632  */
633 static int
634 pipe_loan_alloc(struct pipe *wpipe, int npages)
635 {
636 	vsize_t len;
637 
638 	len = (vsize_t)npages << PAGE_SHIFT;
639 	atomic_add_int(&amountpipekva, len);
640 	wpipe->pipe_map.kva = uvm_km_alloc(kernel_map, len, 0,
641 	    UVM_KMF_VAONLY | UVM_KMF_WAITVA);
642 	if (wpipe->pipe_map.kva == 0) {
643 		atomic_add_int(&amountpipekva, -len);
644 		return (ENOMEM);
645 	}
646 
647 	wpipe->pipe_map.npages = npages;
648 	wpipe->pipe_map.pgs = kmem_alloc(npages * sizeof(struct vm_page *),
649 	    KM_SLEEP);
650 	return (0);
651 }
652 
653 /*
654  * Free resources allocated for loan transfer.
655  */
656 static void
657 pipe_loan_free(struct pipe *wpipe)
658 {
659 	vsize_t len;
660 
661 	len = (vsize_t)wpipe->pipe_map.npages << PAGE_SHIFT;
662 	uvm_emap_remove(wpipe->pipe_map.kva, len);	/* XXX */
663 	uvm_km_free(kernel_map, wpipe->pipe_map.kva, len, UVM_KMF_VAONLY);
664 	wpipe->pipe_map.kva = 0;
665 	atomic_add_int(&amountpipekva, -len);
666 	kmem_free(wpipe->pipe_map.pgs,
667 	    wpipe->pipe_map.npages * sizeof(struct vm_page *));
668 	wpipe->pipe_map.pgs = NULL;
669 }
670 
671 /*
672  * NetBSD direct write, using uvm_loan() mechanism.
673  * This implements the pipe buffer write mechanism.  Note that only
674  * a direct write OR a normal pipe write can be pending at any given time.
675  * If there are any characters in the pipe buffer, the direct write will
676  * be deferred until the receiving process grabs all of the bytes from
677  * the pipe buffer.  Then the direct mapping write is set-up.
678  *
679  * Called with the long-term pipe lock held.
680  */
681 static int
682 pipe_direct_write(file_t *fp, struct pipe *wpipe, struct uio *uio)
683 {
684 	struct vm_page **pgs;
685 	vaddr_t bbase, base, bend;
686 	vsize_t blen, bcnt;
687 	int error, npages;
688 	voff_t bpos;
689 	kmutex_t *lock = wpipe->pipe_lock;
690 
691 	KASSERT(mutex_owned(wpipe->pipe_lock));
692 	KASSERT(wpipe->pipe_map.cnt == 0);
693 
694 	mutex_exit(lock);
695 
696 	/*
697 	 * Handle first PIPE_CHUNK_SIZE bytes of buffer. Deal with buffers
698 	 * not aligned to PAGE_SIZE.
699 	 */
700 	bbase = (vaddr_t)uio->uio_iov->iov_base;
701 	base = trunc_page(bbase);
702 	bend = round_page(bbase + uio->uio_iov->iov_len);
703 	blen = bend - base;
704 	bpos = bbase - base;
705 
706 	if (blen > PIPE_DIRECT_CHUNK) {
707 		blen = PIPE_DIRECT_CHUNK;
708 		bend = base + blen;
709 		bcnt = PIPE_DIRECT_CHUNK - bpos;
710 	} else {
711 		bcnt = uio->uio_iov->iov_len;
712 	}
713 	npages = blen >> PAGE_SHIFT;
714 
715 	/*
716 	 * Free the old kva if we need more pages than we have
717 	 * allocated.
718 	 */
719 	if (wpipe->pipe_map.kva != 0 && npages > wpipe->pipe_map.npages)
720 		pipe_loan_free(wpipe);
721 
722 	/* Allocate new kva. */
723 	if (wpipe->pipe_map.kva == 0) {
724 		error = pipe_loan_alloc(wpipe, npages);
725 		if (error) {
726 			mutex_enter(lock);
727 			return (error);
728 		}
729 	}
730 
731 	/* Loan the write buffer memory from writer process */
732 	pgs = wpipe->pipe_map.pgs;
733 	error = uvm_loan(&uio->uio_vmspace->vm_map, base, blen,
734 			 pgs, UVM_LOAN_TOPAGE);
735 	if (error) {
736 		pipe_loan_free(wpipe);
737 		mutex_enter(lock);
738 		return (ENOMEM); /* so that caller fallback to ordinary write */
739 	}
740 
741 	/* Enter the loaned pages to KVA, produce new emap generation number. */
742 	uvm_emap_enter(wpipe->pipe_map.kva, pgs, npages);
743 	wpipe->pipe_map.egen = uvm_emap_produce();
744 
745 	/* Now we can put the pipe in direct write mode */
746 	wpipe->pipe_map.pos = bpos;
747 	wpipe->pipe_map.cnt = bcnt;
748 
749 	/*
750 	 * But before we can let someone do a direct read, we
751 	 * have to wait until the pipe is drained.  Release the
752 	 * pipe lock while we wait.
753 	 */
754 	mutex_enter(lock);
755 	wpipe->pipe_state |= PIPE_DIRECTW;
756 	pipeunlock(wpipe);
757 
758 	while (error == 0 && wpipe->pipe_buffer.cnt > 0) {
759 		cv_broadcast(&wpipe->pipe_rcv);
760 		error = cv_wait_sig(&wpipe->pipe_wcv, lock);
761 		if (error == 0 && wpipe->pipe_state & PIPE_EOF)
762 			error = EPIPE;
763 	}
764 
765 	/* Pipe is drained; next read will off the direct buffer */
766 	wpipe->pipe_state |= PIPE_DIRECTR;
767 
768 	/* Wait until the reader is done */
769 	while (error == 0 && (wpipe->pipe_state & PIPE_DIRECTR)) {
770 		cv_broadcast(&wpipe->pipe_rcv);
771 		pipeselwakeup(wpipe, wpipe, POLL_IN);
772 		error = cv_wait_sig(&wpipe->pipe_wcv, lock);
773 		if (error == 0 && wpipe->pipe_state & PIPE_EOF)
774 			error = EPIPE;
775 	}
776 
777 	/* Take pipe out of direct write mode */
778 	wpipe->pipe_state &= ~(PIPE_DIRECTW | PIPE_DIRECTR);
779 
780 	/* Acquire the pipe lock and cleanup */
781 	(void)pipelock(wpipe, 0);
782 	mutex_exit(lock);
783 
784 	if (pgs != NULL) {
785 		/* XXX: uvm_emap_remove */
786 		uvm_unloan(pgs, npages, UVM_LOAN_TOPAGE);
787 	}
788 	if (error || amountpipekva > maxpipekva)
789 		pipe_loan_free(wpipe);
790 
791 	mutex_enter(lock);
792 	if (error) {
793 		pipeselwakeup(wpipe, wpipe, POLL_ERR);
794 
795 		/*
796 		 * If nothing was read from what we offered, return error
797 		 * straight on. Otherwise update uio resid first. Caller
798 		 * will deal with the error condition, returning short
799 		 * write, error, or restarting the write(2) as appropriate.
800 		 */
801 		if (wpipe->pipe_map.cnt == bcnt) {
802 			wpipe->pipe_map.cnt = 0;
803 			cv_broadcast(&wpipe->pipe_wcv);
804 			return (error);
805 		}
806 
807 		bcnt -= wpipe->pipe_map.cnt;
808 	}
809 
810 	uio->uio_resid -= bcnt;
811 	/* uio_offset not updated, not set/used for write(2) */
812 	uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + bcnt;
813 	uio->uio_iov->iov_len -= bcnt;
814 	if (uio->uio_iov->iov_len == 0) {
815 		uio->uio_iov++;
816 		uio->uio_iovcnt--;
817 	}
818 
819 	wpipe->pipe_map.cnt = 0;
820 	return (error);
821 }
822 #endif /* !PIPE_NODIRECT */
823 
824 static int
825 pipe_write(file_t *fp, off_t *offset, struct uio *uio, kauth_cred_t cred,
826     int flags)
827 {
828 	struct pipe *wpipe, *rpipe;
829 	struct pipebuf *bp;
830 	kmutex_t *lock;
831 	int error;
832 	unsigned int wakeup_state = 0;
833 
834 	/* We want to write to our peer */
835 	rpipe = (struct pipe *) fp->f_data;
836 	lock = rpipe->pipe_lock;
837 	error = 0;
838 
839 	mutex_enter(lock);
840 	wpipe = rpipe->pipe_peer;
841 
842 	/*
843 	 * Detect loss of pipe read side, issue SIGPIPE if lost.
844 	 */
845 	if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF) != 0) {
846 		mutex_exit(lock);
847 		return EPIPE;
848 	}
849 	++wpipe->pipe_busy;
850 
851 	/* Aquire the long-term pipe lock */
852 	if ((error = pipelock(wpipe, 1)) != 0) {
853 		--wpipe->pipe_busy;
854 		if (wpipe->pipe_busy == 0) {
855 			wpipe->pipe_state &= ~PIPE_RESTART;
856 			cv_broadcast(&wpipe->pipe_draincv);
857 		}
858 		mutex_exit(lock);
859 		return (error);
860 	}
861 
862 	bp = &wpipe->pipe_buffer;
863 
864 	/*
865 	 * If it is advantageous to resize the pipe buffer, do so.
866 	 */
867 	if ((uio->uio_resid > PIPE_SIZE) &&
868 	    (nbigpipe < maxbigpipes) &&
869 #ifndef PIPE_NODIRECT
870 	    (wpipe->pipe_state & PIPE_DIRECTW) == 0 &&
871 #endif
872 	    (bp->size <= PIPE_SIZE) && (bp->cnt == 0)) {
873 
874 		if (pipespace(wpipe, BIG_PIPE_SIZE) == 0)
875 			atomic_inc_uint(&nbigpipe);
876 	}
877 
878 	while (uio->uio_resid) {
879 		size_t space;
880 
881 #ifndef PIPE_NODIRECT
882 		/*
883 		 * Pipe buffered writes cannot be coincidental with
884 		 * direct writes.  Also, only one direct write can be
885 		 * in progress at any one time.  We wait until the currently
886 		 * executing direct write is completed before continuing.
887 		 *
888 		 * We break out if a signal occurs or the reader goes away.
889 		 */
890 		while (error == 0 && wpipe->pipe_state & PIPE_DIRECTW) {
891 			cv_broadcast(&wpipe->pipe_rcv);
892 			pipeunlock(wpipe);
893 			error = cv_wait_sig(&wpipe->pipe_wcv, lock);
894 			(void)pipelock(wpipe, 0);
895 			if (wpipe->pipe_state & PIPE_EOF)
896 				error = EPIPE;
897 		}
898 		if (error)
899 			break;
900 
901 		/*
902 		 * If the transfer is large, we can gain performance if
903 		 * we do process-to-process copies directly.
904 		 * If the write is non-blocking, we don't use the
905 		 * direct write mechanism.
906 		 *
907 		 * The direct write mechanism will detect the reader going
908 		 * away on us.
909 		 */
910 		if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) &&
911 		    (fp->f_flag & FNONBLOCK) == 0 &&
912 		    (wpipe->pipe_map.kva || (amountpipekva < limitpipekva))) {
913 			error = pipe_direct_write(fp, wpipe, uio);
914 
915 			/*
916 			 * Break out if error occurred, unless it's ENOMEM.
917 			 * ENOMEM means we failed to allocate some resources
918 			 * for direct write, so we just fallback to ordinary
919 			 * write. If the direct write was successful,
920 			 * process rest of data via ordinary write.
921 			 */
922 			if (error == 0)
923 				continue;
924 
925 			if (error != ENOMEM)
926 				break;
927 		}
928 #endif /* PIPE_NODIRECT */
929 
930 		space = bp->size - bp->cnt;
931 
932 		/* Writes of size <= PIPE_BUF must be atomic. */
933 		if ((space < uio->uio_resid) && (uio->uio_resid <= PIPE_BUF))
934 			space = 0;
935 
936 		if (space > 0) {
937 			int size;	/* Transfer size */
938 			int segsize;	/* first segment to transfer */
939 
940 			/*
941 			 * Transfer size is minimum of uio transfer
942 			 * and free space in pipe buffer.
943 			 */
944 			if (space > uio->uio_resid)
945 				size = uio->uio_resid;
946 			else
947 				size = space;
948 			/*
949 			 * First segment to transfer is minimum of
950 			 * transfer size and contiguous space in
951 			 * pipe buffer.  If first segment to transfer
952 			 * is less than the transfer size, we've got
953 			 * a wraparound in the buffer.
954 			 */
955 			segsize = bp->size - bp->in;
956 			if (segsize > size)
957 				segsize = size;
958 
959 			/* Transfer first segment */
960 			mutex_exit(lock);
961 			error = uiomove((char *)bp->buffer + bp->in, segsize,
962 			    uio);
963 
964 			if (error == 0 && segsize < size) {
965 				/*
966 				 * Transfer remaining part now, to
967 				 * support atomic writes.  Wraparound
968 				 * happened.
969 				 */
970 				KASSERT(bp->in + segsize == bp->size);
971 				error = uiomove(bp->buffer,
972 				    size - segsize, uio);
973 			}
974 			mutex_enter(lock);
975 			if (error)
976 				break;
977 
978 			bp->in += size;
979 			if (bp->in >= bp->size) {
980 				KASSERT(bp->in == size - segsize + bp->size);
981 				bp->in = size - segsize;
982 			}
983 
984 			bp->cnt += size;
985 			KASSERT(bp->cnt <= bp->size);
986 			wakeup_state = 0;
987 		} else {
988 			/*
989 			 * If the "read-side" has been blocked, wake it up now.
990 			 */
991 			cv_broadcast(&wpipe->pipe_rcv);
992 
993 			/*
994 			 * Don't block on non-blocking I/O.
995 			 */
996 			if (fp->f_flag & FNONBLOCK) {
997 				error = EAGAIN;
998 				break;
999 			}
1000 
1001 			/*
1002 			 * We have no more space and have something to offer,
1003 			 * wake up select/poll.
1004 			 */
1005 			if (bp->cnt)
1006 				pipeselwakeup(wpipe, wpipe, POLL_IN);
1007 
1008 			if (wakeup_state & PIPE_RESTART) {
1009 				error = ERESTART;
1010 				break;
1011 			}
1012 
1013 			pipeunlock(wpipe);
1014 			error = cv_wait_sig(&wpipe->pipe_wcv, lock);
1015 			(void)pipelock(wpipe, 0);
1016 			if (error != 0)
1017 				break;
1018 			/*
1019 			 * If read side wants to go away, we just issue a signal
1020 			 * to ourselves.
1021 			 */
1022 			if (wpipe->pipe_state & PIPE_EOF) {
1023 				error = EPIPE;
1024 				break;
1025 			}
1026 			wakeup_state = wpipe->pipe_state;
1027 		}
1028 	}
1029 
1030 	--wpipe->pipe_busy;
1031 	if (wpipe->pipe_busy == 0) {
1032 		wpipe->pipe_state &= ~PIPE_RESTART;
1033 		cv_broadcast(&wpipe->pipe_draincv);
1034 	}
1035 	if (bp->cnt > 0) {
1036 		cv_broadcast(&wpipe->pipe_rcv);
1037 	}
1038 
1039 	/*
1040 	 * Don't return EPIPE if I/O was successful
1041 	 */
1042 	if (error == EPIPE && bp->cnt == 0 && uio->uio_resid == 0)
1043 		error = 0;
1044 
1045 	if (error == 0)
1046 		getnanotime(&wpipe->pipe_mtime);
1047 
1048 	/*
1049 	 * We have something to offer, wake up select/poll.
1050 	 * wpipe->pipe_map.cnt is always 0 in this point (direct write
1051 	 * is only done synchronously), so check only wpipe->pipe_buffer.cnt
1052 	 */
1053 	if (bp->cnt)
1054 		pipeselwakeup(wpipe, wpipe, POLL_IN);
1055 
1056 	/*
1057 	 * Arrange for next read(2) to do a signal.
1058 	 */
1059 	wpipe->pipe_state |= PIPE_SIGNALR;
1060 
1061 	pipeunlock(wpipe);
1062 	mutex_exit(lock);
1063 	return (error);
1064 }
1065 
1066 /*
1067  * We implement a very minimal set of ioctls for compatibility with sockets.
1068  */
1069 int
1070 pipe_ioctl(file_t *fp, u_long cmd, void *data)
1071 {
1072 	struct pipe *pipe = fp->f_data;
1073 	kmutex_t *lock = pipe->pipe_lock;
1074 
1075 	switch (cmd) {
1076 
1077 	case FIONBIO:
1078 		return (0);
1079 
1080 	case FIOASYNC:
1081 		mutex_enter(lock);
1082 		if (*(int *)data) {
1083 			pipe->pipe_state |= PIPE_ASYNC;
1084 		} else {
1085 			pipe->pipe_state &= ~PIPE_ASYNC;
1086 		}
1087 		mutex_exit(lock);
1088 		return (0);
1089 
1090 	case FIONREAD:
1091 		mutex_enter(lock);
1092 #ifndef PIPE_NODIRECT
1093 		if (pipe->pipe_state & PIPE_DIRECTW)
1094 			*(int *)data = pipe->pipe_map.cnt;
1095 		else
1096 #endif
1097 			*(int *)data = pipe->pipe_buffer.cnt;
1098 		mutex_exit(lock);
1099 		return (0);
1100 
1101 	case FIONWRITE:
1102 		/* Look at other side */
1103 		pipe = pipe->pipe_peer;
1104 		mutex_enter(lock);
1105 #ifndef PIPE_NODIRECT
1106 		if (pipe->pipe_state & PIPE_DIRECTW)
1107 			*(int *)data = pipe->pipe_map.cnt;
1108 		else
1109 #endif
1110 			*(int *)data = pipe->pipe_buffer.cnt;
1111 		mutex_exit(lock);
1112 		return (0);
1113 
1114 	case FIONSPACE:
1115 		/* Look at other side */
1116 		pipe = pipe->pipe_peer;
1117 		mutex_enter(lock);
1118 #ifndef PIPE_NODIRECT
1119 		/*
1120 		 * If we're in direct-mode, we don't really have a
1121 		 * send queue, and any other write will block. Thus
1122 		 * zero seems like the best answer.
1123 		 */
1124 		if (pipe->pipe_state & PIPE_DIRECTW)
1125 			*(int *)data = 0;
1126 		else
1127 #endif
1128 			*(int *)data = pipe->pipe_buffer.size -
1129 			    pipe->pipe_buffer.cnt;
1130 		mutex_exit(lock);
1131 		return (0);
1132 
1133 	case TIOCSPGRP:
1134 	case FIOSETOWN:
1135 		return fsetown(&pipe->pipe_pgid, cmd, data);
1136 
1137 	case TIOCGPGRP:
1138 	case FIOGETOWN:
1139 		return fgetown(pipe->pipe_pgid, cmd, data);
1140 
1141 	}
1142 	return (EPASSTHROUGH);
1143 }
1144 
1145 int
1146 pipe_poll(file_t *fp, int events)
1147 {
1148 	struct pipe *rpipe = fp->f_data;
1149 	struct pipe *wpipe;
1150 	int eof = 0;
1151 	int revents = 0;
1152 
1153 	mutex_enter(rpipe->pipe_lock);
1154 	wpipe = rpipe->pipe_peer;
1155 
1156 	if (events & (POLLIN | POLLRDNORM))
1157 		if ((rpipe->pipe_buffer.cnt > 0) ||
1158 #ifndef PIPE_NODIRECT
1159 		    (rpipe->pipe_state & PIPE_DIRECTR) ||
1160 #endif
1161 		    (rpipe->pipe_state & PIPE_EOF))
1162 			revents |= events & (POLLIN | POLLRDNORM);
1163 
1164 	eof |= (rpipe->pipe_state & PIPE_EOF);
1165 
1166 	if (wpipe == NULL)
1167 		revents |= events & (POLLOUT | POLLWRNORM);
1168 	else {
1169 		if (events & (POLLOUT | POLLWRNORM))
1170 			if ((wpipe->pipe_state & PIPE_EOF) || (
1171 #ifndef PIPE_NODIRECT
1172 			     (wpipe->pipe_state & PIPE_DIRECTW) == 0 &&
1173 #endif
1174 			     (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF))
1175 				revents |= events & (POLLOUT | POLLWRNORM);
1176 
1177 		eof |= (wpipe->pipe_state & PIPE_EOF);
1178 	}
1179 
1180 	if (wpipe == NULL || eof)
1181 		revents |= POLLHUP;
1182 
1183 	if (revents == 0) {
1184 		if (events & (POLLIN | POLLRDNORM))
1185 			selrecord(curlwp, &rpipe->pipe_sel);
1186 
1187 		if (events & (POLLOUT | POLLWRNORM))
1188 			selrecord(curlwp, &wpipe->pipe_sel);
1189 	}
1190 	mutex_exit(rpipe->pipe_lock);
1191 
1192 	return (revents);
1193 }
1194 
1195 static int
1196 pipe_stat(file_t *fp, struct stat *ub)
1197 {
1198 	struct pipe *pipe = fp->f_data;
1199 
1200 	mutex_enter(pipe->pipe_lock);
1201 	memset(ub, 0, sizeof(*ub));
1202 	ub->st_mode = S_IFIFO | S_IRUSR | S_IWUSR;
1203 	ub->st_blksize = pipe->pipe_buffer.size;
1204 	if (ub->st_blksize == 0 && pipe->pipe_peer)
1205 		ub->st_blksize = pipe->pipe_peer->pipe_buffer.size;
1206 	ub->st_size = pipe->pipe_buffer.cnt;
1207 	ub->st_blocks = (ub->st_size) ? 1 : 0;
1208 	ub->st_atimespec = pipe->pipe_atime;
1209 	ub->st_mtimespec = pipe->pipe_mtime;
1210 	ub->st_ctimespec = ub->st_birthtimespec = pipe->pipe_btime;
1211 	ub->st_uid = kauth_cred_geteuid(fp->f_cred);
1212 	ub->st_gid = kauth_cred_getegid(fp->f_cred);
1213 
1214 	/*
1215 	 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen.
1216 	 * XXX (st_dev, st_ino) should be unique.
1217 	 */
1218 	mutex_exit(pipe->pipe_lock);
1219 	return 0;
1220 }
1221 
1222 static int
1223 pipe_close(file_t *fp)
1224 {
1225 	struct pipe *pipe = fp->f_data;
1226 
1227 	fp->f_data = NULL;
1228 	pipeclose(pipe);
1229 	return (0);
1230 }
1231 
1232 static void
1233 pipe_restart(file_t *fp)
1234 {
1235 	struct pipe *pipe = fp->f_data;
1236 
1237 	/*
1238 	 * Unblock blocked reads/writes in order to allow close() to complete.
1239 	 * System calls return ERESTART so that the fd is revalidated.
1240 	 * (Partial writes return the transfer length.)
1241 	 */
1242 	mutex_enter(pipe->pipe_lock);
1243 	pipe->pipe_state |= PIPE_RESTART;
1244 	/* Wakeup both cvs, maybe we only need one, but maybe there are some
1245 	 * other paths where wakeup is needed, and it saves deciding which! */
1246 	cv_broadcast(&pipe->pipe_rcv);
1247 	cv_broadcast(&pipe->pipe_wcv);
1248 	mutex_exit(pipe->pipe_lock);
1249 }
1250 
1251 static void
1252 pipe_free_kmem(struct pipe *pipe)
1253 {
1254 
1255 	if (pipe->pipe_buffer.buffer != NULL) {
1256 		if (pipe->pipe_buffer.size > PIPE_SIZE) {
1257 			atomic_dec_uint(&nbigpipe);
1258 		}
1259 		if (pipe->pipe_buffer.buffer != (void *)pipe->pipe_kmem) {
1260 			uvm_km_free(kernel_map,
1261 			    (vaddr_t)pipe->pipe_buffer.buffer,
1262 			    pipe->pipe_buffer.size, UVM_KMF_PAGEABLE);
1263 			atomic_add_int(&amountpipekva,
1264 			    -pipe->pipe_buffer.size);
1265 		}
1266 		pipe->pipe_buffer.buffer = NULL;
1267 	}
1268 #ifndef PIPE_NODIRECT
1269 	if (pipe->pipe_map.kva != 0) {
1270 		pipe_loan_free(pipe);
1271 		pipe->pipe_map.cnt = 0;
1272 		pipe->pipe_map.kva = 0;
1273 		pipe->pipe_map.pos = 0;
1274 		pipe->pipe_map.npages = 0;
1275 	}
1276 #endif /* !PIPE_NODIRECT */
1277 }
1278 
1279 /*
1280  * Shutdown the pipe.
1281  */
1282 static void
1283 pipeclose(struct pipe *pipe)
1284 {
1285 	kmutex_t *lock;
1286 	struct pipe *ppipe;
1287 
1288 	if (pipe == NULL)
1289 		return;
1290 
1291 	KASSERT(cv_is_valid(&pipe->pipe_rcv));
1292 	KASSERT(cv_is_valid(&pipe->pipe_wcv));
1293 	KASSERT(cv_is_valid(&pipe->pipe_draincv));
1294 	KASSERT(cv_is_valid(&pipe->pipe_lkcv));
1295 
1296 	lock = pipe->pipe_lock;
1297 	if (lock == NULL)
1298 		/* Must have failed during create */
1299 		goto free_resources;
1300 
1301 	mutex_enter(lock);
1302 	pipeselwakeup(pipe, pipe, POLL_HUP);
1303 
1304 	/*
1305 	 * If the other side is blocked, wake it up saying that
1306 	 * we want to close it down.
1307 	 */
1308 	pipe->pipe_state |= PIPE_EOF;
1309 	if (pipe->pipe_busy) {
1310 		while (pipe->pipe_busy) {
1311 			cv_broadcast(&pipe->pipe_wcv);
1312 			cv_wait_sig(&pipe->pipe_draincv, lock);
1313 		}
1314 	}
1315 
1316 	/*
1317 	 * Disconnect from peer.
1318 	 */
1319 	if ((ppipe = pipe->pipe_peer) != NULL) {
1320 		pipeselwakeup(ppipe, ppipe, POLL_HUP);
1321 		ppipe->pipe_state |= PIPE_EOF;
1322 		cv_broadcast(&ppipe->pipe_rcv);
1323 		ppipe->pipe_peer = NULL;
1324 	}
1325 
1326 	/*
1327 	 * Any knote objects still left in the list are
1328 	 * the one attached by peer.  Since no one will
1329 	 * traverse this list, we just clear it.
1330 	 */
1331 	SLIST_INIT(&pipe->pipe_sel.sel_klist);
1332 
1333 	KASSERT((pipe->pipe_state & PIPE_LOCKFL) == 0);
1334 	mutex_exit(lock);
1335 	mutex_obj_free(lock);
1336 
1337 	/*
1338 	 * Free resources.
1339 	 */
1340     free_resources:
1341 	pipe->pipe_pgid = 0;
1342 	pipe->pipe_state = PIPE_SIGNALR;
1343 	pipe_free_kmem(pipe);
1344 	if (pipe->pipe_kmem != 0) {
1345 		pool_cache_put(pipe_rd_cache, pipe);
1346 	} else {
1347 		pool_cache_put(pipe_wr_cache, pipe);
1348 	}
1349 }
1350 
1351 static void
1352 filt_pipedetach(struct knote *kn)
1353 {
1354 	struct pipe *pipe;
1355 	kmutex_t *lock;
1356 
1357 	pipe = ((file_t *)kn->kn_obj)->f_data;
1358 	lock = pipe->pipe_lock;
1359 
1360 	mutex_enter(lock);
1361 
1362 	switch(kn->kn_filter) {
1363 	case EVFILT_WRITE:
1364 		/* Need the peer structure, not our own. */
1365 		pipe = pipe->pipe_peer;
1366 
1367 		/* If reader end already closed, just return. */
1368 		if (pipe == NULL) {
1369 			mutex_exit(lock);
1370 			return;
1371 		}
1372 
1373 		break;
1374 	default:
1375 		/* Nothing to do. */
1376 		break;
1377 	}
1378 
1379 	KASSERT(kn->kn_hook == pipe);
1380 	SLIST_REMOVE(&pipe->pipe_sel.sel_klist, kn, knote, kn_selnext);
1381 	mutex_exit(lock);
1382 }
1383 
1384 static int
1385 filt_piperead(struct knote *kn, long hint)
1386 {
1387 	struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_data;
1388 	struct pipe *wpipe;
1389 
1390 	if ((hint & NOTE_SUBMIT) == 0) {
1391 		mutex_enter(rpipe->pipe_lock);
1392 	}
1393 	wpipe = rpipe->pipe_peer;
1394 	kn->kn_data = rpipe->pipe_buffer.cnt;
1395 
1396 	if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW))
1397 		kn->kn_data = rpipe->pipe_map.cnt;
1398 
1399 	if ((rpipe->pipe_state & PIPE_EOF) ||
1400 	    (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) {
1401 		kn->kn_flags |= EV_EOF;
1402 		if ((hint & NOTE_SUBMIT) == 0) {
1403 			mutex_exit(rpipe->pipe_lock);
1404 		}
1405 		return (1);
1406 	}
1407 
1408 	if ((hint & NOTE_SUBMIT) == 0) {
1409 		mutex_exit(rpipe->pipe_lock);
1410 	}
1411 	return (kn->kn_data > 0);
1412 }
1413 
1414 static int
1415 filt_pipewrite(struct knote *kn, long hint)
1416 {
1417 	struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_data;
1418 	struct pipe *wpipe;
1419 
1420 	if ((hint & NOTE_SUBMIT) == 0) {
1421 		mutex_enter(rpipe->pipe_lock);
1422 	}
1423 	wpipe = rpipe->pipe_peer;
1424 
1425 	if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) {
1426 		kn->kn_data = 0;
1427 		kn->kn_flags |= EV_EOF;
1428 		if ((hint & NOTE_SUBMIT) == 0) {
1429 			mutex_exit(rpipe->pipe_lock);
1430 		}
1431 		return (1);
1432 	}
1433 	kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt;
1434 	if (wpipe->pipe_state & PIPE_DIRECTW)
1435 		kn->kn_data = 0;
1436 
1437 	if ((hint & NOTE_SUBMIT) == 0) {
1438 		mutex_exit(rpipe->pipe_lock);
1439 	}
1440 	return (kn->kn_data >= PIPE_BUF);
1441 }
1442 
1443 static const struct filterops pipe_rfiltops =
1444 	{ 1, NULL, filt_pipedetach, filt_piperead };
1445 static const struct filterops pipe_wfiltops =
1446 	{ 1, NULL, filt_pipedetach, filt_pipewrite };
1447 
1448 static int
1449 pipe_kqfilter(file_t *fp, struct knote *kn)
1450 {
1451 	struct pipe *pipe;
1452 	kmutex_t *lock;
1453 
1454 	pipe = ((file_t *)kn->kn_obj)->f_data;
1455 	lock = pipe->pipe_lock;
1456 
1457 	mutex_enter(lock);
1458 
1459 	switch (kn->kn_filter) {
1460 	case EVFILT_READ:
1461 		kn->kn_fop = &pipe_rfiltops;
1462 		break;
1463 	case EVFILT_WRITE:
1464 		kn->kn_fop = &pipe_wfiltops;
1465 		pipe = pipe->pipe_peer;
1466 		if (pipe == NULL) {
1467 			/* Other end of pipe has been closed. */
1468 			mutex_exit(lock);
1469 			return (EBADF);
1470 		}
1471 		break;
1472 	default:
1473 		mutex_exit(lock);
1474 		return (EINVAL);
1475 	}
1476 
1477 	kn->kn_hook = pipe;
1478 	SLIST_INSERT_HEAD(&pipe->pipe_sel.sel_klist, kn, kn_selnext);
1479 	mutex_exit(lock);
1480 
1481 	return (0);
1482 }
1483 
1484 /*
1485  * Handle pipe sysctls.
1486  */
1487 SYSCTL_SETUP(sysctl_kern_pipe_setup, "sysctl kern.pipe subtree setup")
1488 {
1489 
1490 	sysctl_createv(clog, 0, NULL, NULL,
1491 		       CTLFLAG_PERMANENT,
1492 		       CTLTYPE_NODE, "kern", NULL,
1493 		       NULL, 0, NULL, 0,
1494 		       CTL_KERN, CTL_EOL);
1495 	sysctl_createv(clog, 0, NULL, NULL,
1496 		       CTLFLAG_PERMANENT,
1497 		       CTLTYPE_NODE, "pipe",
1498 		       SYSCTL_DESCR("Pipe settings"),
1499 		       NULL, 0, NULL, 0,
1500 		       CTL_KERN, KERN_PIPE, CTL_EOL);
1501 
1502 	sysctl_createv(clog, 0, NULL, NULL,
1503 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1504 		       CTLTYPE_INT, "maxkvasz",
1505 		       SYSCTL_DESCR("Maximum amount of kernel memory to be "
1506 				    "used for pipes"),
1507 		       NULL, 0, &maxpipekva, 0,
1508 		       CTL_KERN, KERN_PIPE, KERN_PIPE_MAXKVASZ, CTL_EOL);
1509 	sysctl_createv(clog, 0, NULL, NULL,
1510 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1511 		       CTLTYPE_INT, "maxloankvasz",
1512 		       SYSCTL_DESCR("Limit for direct transfers via page loan"),
1513 		       NULL, 0, &limitpipekva, 0,
1514 		       CTL_KERN, KERN_PIPE, KERN_PIPE_LIMITKVA, CTL_EOL);
1515 	sysctl_createv(clog, 0, NULL, NULL,
1516 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1517 		       CTLTYPE_INT, "maxbigpipes",
1518 		       SYSCTL_DESCR("Maximum number of \"big\" pipes"),
1519 		       NULL, 0, &maxbigpipes, 0,
1520 		       CTL_KERN, KERN_PIPE, KERN_PIPE_MAXBIGPIPES, CTL_EOL);
1521 	sysctl_createv(clog, 0, NULL, NULL,
1522 		       CTLFLAG_PERMANENT,
1523 		       CTLTYPE_INT, "nbigpipes",
1524 		       SYSCTL_DESCR("Number of \"big\" pipes"),
1525 		       NULL, 0, &nbigpipe, 0,
1526 		       CTL_KERN, KERN_PIPE, KERN_PIPE_NBIGPIPES, CTL_EOL);
1527 	sysctl_createv(clog, 0, NULL, NULL,
1528 		       CTLFLAG_PERMANENT,
1529 		       CTLTYPE_INT, "kvasize",
1530 		       SYSCTL_DESCR("Amount of kernel memory consumed by pipe "
1531 				    "buffers"),
1532 		       NULL, 0, &amountpipekva, 0,
1533 		       CTL_KERN, KERN_PIPE, KERN_PIPE_KVASIZE, CTL_EOL);
1534 }
1535