1 /* $NetBSD: sys_pipe.c,v 1.95 2008/01/28 20:01:50 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2003, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Paul Kranenburg, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Copyright (c) 1996 John S. Dyson 41 * All rights reserved. 42 * 43 * Redistribution and use in source and binary forms, with or without 44 * modification, are permitted provided that the following conditions 45 * are met: 46 * 1. Redistributions of source code must retain the above copyright 47 * notice immediately at the beginning of the file, without modification, 48 * this list of conditions, and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. Absolutely no warranty of function or purpose is made by the author 53 * John S. Dyson. 54 * 4. Modifications may be freely made to this file if the above conditions 55 * are met. 56 * 57 * $FreeBSD: src/sys/kern/sys_pipe.c,v 1.95 2002/03/09 22:06:31 alfred Exp $ 58 */ 59 60 /* 61 * This file contains a high-performance replacement for the socket-based 62 * pipes scheme originally used in FreeBSD/4.4Lite. It does not support 63 * all features of sockets, but does do everything that pipes normally 64 * do. 65 * 66 * Adaption for NetBSD UVM, including uvm_loan() based direct write, was 67 * written by Jaromir Dolecek. 68 */ 69 70 /* 71 * This code has two modes of operation, a small write mode and a large 72 * write mode. The small write mode acts like conventional pipes with 73 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 74 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 75 * and PIPE_SIZE in size it is mapped read-only into the kernel address space 76 * using the UVM page loan facility from where the receiving process can copy 77 * the data directly from the pages in the sending process. 78 * 79 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 80 * happen for small transfers so that the system will not spend all of 81 * its time context switching. PIPE_SIZE is constrained by the 82 * amount of kernel virtual memory. 83 */ 84 85 #include <sys/cdefs.h> 86 __KERNEL_RCSID(0, "$NetBSD: sys_pipe.c,v 1.95 2008/01/28 20:01:50 ad Exp $"); 87 88 #include <sys/param.h> 89 #include <sys/systm.h> 90 #include <sys/proc.h> 91 #include <sys/fcntl.h> 92 #include <sys/file.h> 93 #include <sys/filedesc.h> 94 #include <sys/filio.h> 95 #include <sys/kernel.h> 96 #include <sys/ttycom.h> 97 #include <sys/stat.h> 98 #include <sys/malloc.h> 99 #include <sys/poll.h> 100 #include <sys/signalvar.h> 101 #include <sys/vnode.h> 102 #include <sys/uio.h> 103 #include <sys/select.h> 104 #include <sys/mount.h> 105 #include <sys/syscallargs.h> 106 #include <sys/sysctl.h> 107 #include <sys/kauth.h> 108 #include <sys/atomic.h> 109 #include <sys/pipe.h> 110 111 #include <uvm/uvm.h> 112 113 /* 114 * Use this define if you want to disable *fancy* VM things. Expect an 115 * approx 30% decrease in transfer rate. 116 */ 117 /* #define PIPE_NODIRECT */ 118 119 /* 120 * interfaces to the outside world 121 */ 122 static int pipe_read(struct file *fp, off_t *offset, struct uio *uio, 123 kauth_cred_t cred, int flags); 124 static int pipe_write(struct file *fp, off_t *offset, struct uio *uio, 125 kauth_cred_t cred, int flags); 126 static int pipe_close(struct file *fp, struct lwp *l); 127 static int pipe_poll(struct file *fp, int events, struct lwp *l); 128 static int pipe_kqfilter(struct file *fp, struct knote *kn); 129 static int pipe_stat(struct file *fp, struct stat *sb, struct lwp *l); 130 static int pipe_ioctl(struct file *fp, u_long cmd, void *data, 131 struct lwp *l); 132 133 static const struct fileops pipeops = { 134 pipe_read, pipe_write, pipe_ioctl, fnullop_fcntl, pipe_poll, 135 pipe_stat, pipe_close, pipe_kqfilter 136 }; 137 138 /* 139 * Single mutex shared between both ends of the pipe. 140 */ 141 142 struct pipe_mutex { 143 kmutex_t pm_mutex; 144 u_int pm_refcnt; 145 }; 146 147 /* 148 * Default pipe buffer size(s), this can be kind-of large now because pipe 149 * space is pageable. The pipe code will try to maintain locality of 150 * reference for performance reasons, so small amounts of outstanding I/O 151 * will not wipe the cache. 152 */ 153 #define MINPIPESIZE (PIPE_SIZE/3) 154 #define MAXPIPESIZE (2*PIPE_SIZE/3) 155 156 /* 157 * Maximum amount of kva for pipes -- this is kind-of a soft limit, but 158 * is there so that on large systems, we don't exhaust it. 159 */ 160 #define MAXPIPEKVA (8*1024*1024) 161 static u_int maxpipekva = MAXPIPEKVA; 162 163 /* 164 * Limit for direct transfers, we cannot, of course limit 165 * the amount of kva for pipes in general though. 166 */ 167 #define LIMITPIPEKVA (16*1024*1024) 168 static u_int limitpipekva = LIMITPIPEKVA; 169 170 /* 171 * Limit the number of "big" pipes 172 */ 173 #define LIMITBIGPIPES 32 174 static u_int maxbigpipes = LIMITBIGPIPES; 175 static u_int nbigpipe = 0; 176 177 /* 178 * Amount of KVA consumed by pipe buffers. 179 */ 180 static u_int amountpipekva = 0; 181 182 MALLOC_DEFINE(M_PIPE, "pipe", "Pipe structures"); 183 184 static void pipeclose(struct file *fp, struct pipe *pipe); 185 static void pipe_free_kmem(struct pipe *pipe); 186 static int pipe_create(struct pipe **pipep, int allockva, struct pipe_mutex *); 187 static int pipelock(struct pipe *pipe, int catch); 188 static inline void pipeunlock(struct pipe *pipe); 189 static void pipeselwakeup(struct pipe *pipe, struct pipe *sigp, int code); 190 #ifndef PIPE_NODIRECT 191 static int pipe_direct_write(struct file *fp, struct pipe *wpipe, 192 struct uio *uio); 193 #endif 194 static int pipespace(struct pipe *pipe, int size); 195 196 #ifndef PIPE_NODIRECT 197 static int pipe_loan_alloc(struct pipe *, int); 198 static void pipe_loan_free(struct pipe *); 199 #endif /* PIPE_NODIRECT */ 200 201 static int pipe_mutex_ctor(void *, void *, int); 202 static void pipe_mutex_dtor(void *, void *); 203 204 static pool_cache_t pipe_cache; 205 static pool_cache_t pipe_mutex_cache; 206 207 void 208 pipe_init(void) 209 { 210 size_t size; 211 212 pipe_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "pipepl", 213 NULL, IPL_NONE, NULL, NULL, NULL); 214 KASSERT(pipe_cache != NULL); 215 216 size = (sizeof(struct pipe_mutex) + (CACHE_LINE_SIZE - 1)) & 217 (CACHE_LINE_SIZE - 1); 218 pipe_mutex_cache = pool_cache_init(size, CACHE_LINE_SIZE, 219 0, 0, "pipemtxpl", NULL, IPL_NONE, pipe_mutex_ctor, 220 pipe_mutex_dtor, NULL); 221 KASSERT(pipe_cache != NULL); 222 } 223 224 static int 225 pipe_mutex_ctor(void *arg, void *obj, int flag) 226 { 227 struct pipe_mutex *pm = obj; 228 229 mutex_init(&pm->pm_mutex, MUTEX_DEFAULT, IPL_NONE); 230 pm->pm_refcnt = 0; 231 232 return 0; 233 } 234 235 static void 236 pipe_mutex_dtor(void *arg, void *obj) 237 { 238 struct pipe_mutex *pm = obj; 239 240 KASSERT(pm->pm_refcnt == 0); 241 242 mutex_destroy(&pm->pm_mutex); 243 } 244 245 /* 246 * The pipe system call for the DTYPE_PIPE type of pipes 247 */ 248 249 /* ARGSUSED */ 250 int 251 sys_pipe(struct lwp *l, const void *v, register_t *retval) 252 { 253 struct file *rf, *wf; 254 struct pipe *rpipe, *wpipe; 255 struct pipe_mutex *mutex; 256 int fd, error; 257 258 rpipe = wpipe = NULL; 259 mutex = pool_cache_get(pipe_mutex_cache, PR_WAITOK); 260 if (mutex == NULL) 261 return (ENOMEM); 262 if (pipe_create(&rpipe, 1, mutex) || pipe_create(&wpipe, 0, mutex)) { 263 pipeclose(NULL, rpipe); 264 pipeclose(NULL, wpipe); 265 return (ENFILE); 266 } 267 268 /* 269 * Note: the file structure returned from falloc() is marked 270 * as 'larval' initially. Unless we mark it as 'mature' by 271 * FILE_SET_MATURE(), any attempt to do anything with it would 272 * return EBADF, including e.g. dup(2) or close(2). This avoids 273 * file descriptor races if we block in the second falloc(). 274 */ 275 276 error = falloc(l, &rf, &fd); 277 if (error) 278 goto free2; 279 retval[0] = fd; 280 rf->f_flag = FREAD; 281 rf->f_type = DTYPE_PIPE; 282 rf->f_data = (void *)rpipe; 283 rf->f_ops = &pipeops; 284 285 error = falloc(l, &wf, &fd); 286 if (error) 287 goto free3; 288 retval[1] = fd; 289 wf->f_flag = FWRITE; 290 wf->f_type = DTYPE_PIPE; 291 wf->f_data = (void *)wpipe; 292 wf->f_ops = &pipeops; 293 294 rpipe->pipe_peer = wpipe; 295 wpipe->pipe_peer = rpipe; 296 297 FILE_SET_MATURE(rf); 298 FILE_SET_MATURE(wf); 299 FILE_UNUSE(rf, l); 300 FILE_UNUSE(wf, l); 301 return (0); 302 free3: 303 FILE_UNUSE(rf, l); 304 ffree(rf); 305 fdremove(l->l_proc->p_fd, retval[0]); 306 free2: 307 pipeclose(NULL, wpipe); 308 pipeclose(NULL, rpipe); 309 310 return (error); 311 } 312 313 /* 314 * Allocate kva for pipe circular buffer, the space is pageable 315 * This routine will 'realloc' the size of a pipe safely, if it fails 316 * it will retain the old buffer. 317 * If it fails it will return ENOMEM. 318 */ 319 static int 320 pipespace(struct pipe *pipe, int size) 321 { 322 void *buffer; 323 /* 324 * Allocate pageable virtual address space. Physical memory is 325 * allocated on demand. 326 */ 327 buffer = (void *) uvm_km_alloc(kernel_map, round_page(size), 0, 328 UVM_KMF_PAGEABLE); 329 if (buffer == NULL) 330 return (ENOMEM); 331 332 /* free old resources if we're resizing */ 333 pipe_free_kmem(pipe); 334 pipe->pipe_buffer.buffer = buffer; 335 pipe->pipe_buffer.size = size; 336 pipe->pipe_buffer.in = 0; 337 pipe->pipe_buffer.out = 0; 338 pipe->pipe_buffer.cnt = 0; 339 atomic_add_int(&amountpipekva, pipe->pipe_buffer.size); 340 return (0); 341 } 342 343 /* 344 * Initialize and allocate VM and memory for pipe. 345 */ 346 static int 347 pipe_create(struct pipe **pipep, int allockva, struct pipe_mutex *mutex) 348 { 349 struct pipe *pipe; 350 int error; 351 352 pipe = *pipep = pool_cache_get(pipe_cache, PR_WAITOK); 353 mutex->pm_refcnt++; 354 355 /* Initialize */ 356 memset(pipe, 0, sizeof(struct pipe)); 357 pipe->pipe_state = PIPE_SIGNALR; 358 359 getmicrotime(&pipe->pipe_ctime); 360 pipe->pipe_atime = pipe->pipe_ctime; 361 pipe->pipe_mtime = pipe->pipe_ctime; 362 pipe->pipe_lock = &mutex->pm_mutex; 363 cv_init(&pipe->pipe_cv, "pipe"); 364 cv_init(&pipe->pipe_lkcv, "pipelk"); 365 selinit(&pipe->pipe_sel); 366 367 if (allockva && (error = pipespace(pipe, PIPE_SIZE))) 368 return (error); 369 370 return (0); 371 } 372 373 374 /* 375 * Lock a pipe for I/O, blocking other access 376 * Called with pipe spin lock held. 377 * Return with pipe spin lock released on success. 378 */ 379 static int 380 pipelock(struct pipe *pipe, int catch) 381 { 382 int error; 383 384 KASSERT(mutex_owned(pipe->pipe_lock)); 385 386 while (pipe->pipe_state & PIPE_LOCKFL) { 387 pipe->pipe_state |= PIPE_LWANT; 388 if (catch) { 389 error = cv_wait_sig(&pipe->pipe_lkcv, pipe->pipe_lock); 390 if (error != 0) 391 return error; 392 } else 393 cv_wait(&pipe->pipe_lkcv, pipe->pipe_lock); 394 } 395 396 pipe->pipe_state |= PIPE_LOCKFL; 397 398 return 0; 399 } 400 401 /* 402 * unlock a pipe I/O lock 403 */ 404 static inline void 405 pipeunlock(struct pipe *pipe) 406 { 407 408 KASSERT(pipe->pipe_state & PIPE_LOCKFL); 409 410 pipe->pipe_state &= ~PIPE_LOCKFL; 411 if (pipe->pipe_state & PIPE_LWANT) { 412 pipe->pipe_state &= ~PIPE_LWANT; 413 cv_broadcast(&pipe->pipe_lkcv); 414 } 415 } 416 417 /* 418 * Select/poll wakup. This also sends SIGIO to peer connected to 419 * 'sigpipe' side of pipe. 420 */ 421 static void 422 pipeselwakeup(struct pipe *selp, struct pipe *sigp, int code) 423 { 424 int band; 425 426 selnotify(&selp->pipe_sel, NOTE_SUBMIT); 427 428 if (sigp == NULL || (sigp->pipe_state & PIPE_ASYNC) == 0) 429 return; 430 431 switch (code) { 432 case POLL_IN: 433 band = POLLIN|POLLRDNORM; 434 break; 435 case POLL_OUT: 436 band = POLLOUT|POLLWRNORM; 437 break; 438 case POLL_HUP: 439 band = POLLHUP; 440 break; 441 #if POLL_HUP != POLL_ERR 442 case POLL_ERR: 443 band = POLLERR; 444 break; 445 #endif 446 default: 447 band = 0; 448 #ifdef DIAGNOSTIC 449 printf("bad siginfo code %d in pipe notification.\n", code); 450 #endif 451 break; 452 } 453 454 fownsignal(sigp->pipe_pgid, SIGIO, code, band, selp); 455 } 456 457 /* ARGSUSED */ 458 static int 459 pipe_read(struct file *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 460 int flags) 461 { 462 struct pipe *rpipe = (struct pipe *) fp->f_data; 463 struct pipebuf *bp = &rpipe->pipe_buffer; 464 kmutex_t *lock = rpipe->pipe_lock; 465 int error; 466 size_t nread = 0; 467 size_t size; 468 size_t ocnt; 469 470 mutex_enter(lock); 471 ++rpipe->pipe_busy; 472 ocnt = bp->cnt; 473 474 again: 475 error = pipelock(rpipe, 1); 476 if (error) 477 goto unlocked_error; 478 479 while (uio->uio_resid) { 480 /* 481 * normal pipe buffer receive 482 */ 483 if (bp->cnt > 0) { 484 size = bp->size - bp->out; 485 if (size > bp->cnt) 486 size = bp->cnt; 487 if (size > uio->uio_resid) 488 size = uio->uio_resid; 489 490 mutex_exit(lock); 491 error = uiomove((char *)bp->buffer + bp->out, size, uio); 492 mutex_enter(lock); 493 if (error) 494 break; 495 496 bp->out += size; 497 if (bp->out >= bp->size) 498 bp->out = 0; 499 500 bp->cnt -= size; 501 502 /* 503 * If there is no more to read in the pipe, reset 504 * its pointers to the beginning. This improves 505 * cache hit stats. 506 */ 507 if (bp->cnt == 0) { 508 bp->in = 0; 509 bp->out = 0; 510 } 511 nread += size; 512 continue; 513 } 514 515 #ifndef PIPE_NODIRECT 516 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) { 517 /* 518 * Direct copy, bypassing a kernel buffer. 519 */ 520 void * va; 521 522 KASSERT(rpipe->pipe_state & PIPE_DIRECTW); 523 524 size = rpipe->pipe_map.cnt; 525 if (size > uio->uio_resid) 526 size = uio->uio_resid; 527 528 va = (char *)rpipe->pipe_map.kva + rpipe->pipe_map.pos; 529 mutex_exit(lock); 530 error = uiomove(va, size, uio); 531 mutex_enter(lock); 532 if (error) 533 break; 534 nread += size; 535 rpipe->pipe_map.pos += size; 536 rpipe->pipe_map.cnt -= size; 537 if (rpipe->pipe_map.cnt == 0) { 538 rpipe->pipe_state &= ~PIPE_DIRECTR; 539 cv_broadcast(&rpipe->pipe_cv); 540 } 541 continue; 542 } 543 #endif 544 /* 545 * Break if some data was read. 546 */ 547 if (nread > 0) 548 break; 549 550 /* 551 * detect EOF condition 552 * read returns 0 on EOF, no need to set error 553 */ 554 if (rpipe->pipe_state & PIPE_EOF) 555 break; 556 557 /* 558 * don't block on non-blocking I/O 559 */ 560 if (fp->f_flag & FNONBLOCK) { 561 error = EAGAIN; 562 break; 563 } 564 565 /* 566 * Unlock the pipe buffer for our remaining processing. 567 * We will either break out with an error or we will 568 * sleep and relock to loop. 569 */ 570 pipeunlock(rpipe); 571 572 /* 573 * Re-check to see if more direct writes are pending. 574 */ 575 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) 576 goto again; 577 578 /* 579 * We want to read more, wake up select/poll. 580 */ 581 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_IN); 582 583 /* 584 * If the "write-side" is blocked, wake it up now. 585 */ 586 cv_broadcast(&rpipe->pipe_cv); 587 588 /* Now wait until the pipe is filled */ 589 error = cv_wait_sig(&rpipe->pipe_cv, lock); 590 if (error != 0) 591 goto unlocked_error; 592 goto again; 593 } 594 595 if (error == 0) 596 getmicrotime(&rpipe->pipe_atime); 597 pipeunlock(rpipe); 598 599 unlocked_error: 600 --rpipe->pipe_busy; 601 if (rpipe->pipe_busy == 0 || bp->cnt < MINPIPESIZE) { 602 cv_broadcast(&rpipe->pipe_cv); 603 } 604 605 /* 606 * If anything was read off the buffer, signal to the writer it's 607 * possible to write more data. Also send signal if we are here for the 608 * first time after last write. 609 */ 610 if ((bp->size - bp->cnt) >= PIPE_BUF 611 && (ocnt != bp->cnt || (rpipe->pipe_state & PIPE_SIGNALR))) { 612 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 613 rpipe->pipe_state &= ~PIPE_SIGNALR; 614 } 615 616 mutex_exit(lock); 617 return (error); 618 } 619 620 #ifndef PIPE_NODIRECT 621 /* 622 * Allocate structure for loan transfer. 623 */ 624 static int 625 pipe_loan_alloc(struct pipe *wpipe, int npages) 626 { 627 vsize_t len; 628 629 len = (vsize_t)npages << PAGE_SHIFT; 630 atomic_add_int(&amountpipekva, len); 631 wpipe->pipe_map.kva = uvm_km_alloc(kernel_map, len, 0, 632 UVM_KMF_VAONLY | UVM_KMF_WAITVA); 633 if (wpipe->pipe_map.kva == 0) { 634 atomic_add_int(&amountpipekva, -len); 635 return (ENOMEM); 636 } 637 638 wpipe->pipe_map.npages = npages; 639 wpipe->pipe_map.pgs = malloc(npages * sizeof(struct vm_page *), M_PIPE, 640 M_WAITOK); 641 return (0); 642 } 643 644 /* 645 * Free resources allocated for loan transfer. 646 */ 647 static void 648 pipe_loan_free(struct pipe *wpipe) 649 { 650 vsize_t len; 651 652 len = (vsize_t)wpipe->pipe_map.npages << PAGE_SHIFT; 653 uvm_km_free(kernel_map, wpipe->pipe_map.kva, len, UVM_KMF_VAONLY); 654 wpipe->pipe_map.kva = 0; 655 atomic_add_int(&amountpipekva, -len); 656 free(wpipe->pipe_map.pgs, M_PIPE); 657 wpipe->pipe_map.pgs = NULL; 658 } 659 660 /* 661 * NetBSD direct write, using uvm_loan() mechanism. 662 * This implements the pipe buffer write mechanism. Note that only 663 * a direct write OR a normal pipe write can be pending at any given time. 664 * If there are any characters in the pipe buffer, the direct write will 665 * be deferred until the receiving process grabs all of the bytes from 666 * the pipe buffer. Then the direct mapping write is set-up. 667 * 668 * Called with the long-term pipe lock held. 669 */ 670 static int 671 pipe_direct_write(struct file *fp, struct pipe *wpipe, struct uio *uio) 672 { 673 int error, npages, j; 674 struct vm_page **pgs; 675 vaddr_t bbase, kva, base, bend; 676 vsize_t blen, bcnt; 677 voff_t bpos; 678 kmutex_t *lock = wpipe->pipe_lock; 679 680 KASSERT(mutex_owned(wpipe->pipe_lock)); 681 KASSERT(wpipe->pipe_map.cnt == 0); 682 683 mutex_exit(lock); 684 685 /* 686 * Handle first PIPE_CHUNK_SIZE bytes of buffer. Deal with buffers 687 * not aligned to PAGE_SIZE. 688 */ 689 bbase = (vaddr_t)uio->uio_iov->iov_base; 690 base = trunc_page(bbase); 691 bend = round_page(bbase + uio->uio_iov->iov_len); 692 blen = bend - base; 693 bpos = bbase - base; 694 695 if (blen > PIPE_DIRECT_CHUNK) { 696 blen = PIPE_DIRECT_CHUNK; 697 bend = base + blen; 698 bcnt = PIPE_DIRECT_CHUNK - bpos; 699 } else { 700 bcnt = uio->uio_iov->iov_len; 701 } 702 npages = blen >> PAGE_SHIFT; 703 704 /* 705 * Free the old kva if we need more pages than we have 706 * allocated. 707 */ 708 if (wpipe->pipe_map.kva != 0 && npages > wpipe->pipe_map.npages) 709 pipe_loan_free(wpipe); 710 711 /* Allocate new kva. */ 712 if (wpipe->pipe_map.kva == 0) { 713 error = pipe_loan_alloc(wpipe, npages); 714 if (error) { 715 mutex_enter(lock); 716 return (error); 717 } 718 } 719 720 /* Loan the write buffer memory from writer process */ 721 pgs = wpipe->pipe_map.pgs; 722 error = uvm_loan(&uio->uio_vmspace->vm_map, base, blen, 723 pgs, UVM_LOAN_TOPAGE); 724 if (error) { 725 pipe_loan_free(wpipe); 726 mutex_enter(lock); 727 return (ENOMEM); /* so that caller fallback to ordinary write */ 728 } 729 730 /* Enter the loaned pages to kva */ 731 kva = wpipe->pipe_map.kva; 732 for (j = 0; j < npages; j++, kva += PAGE_SIZE) { 733 pmap_kenter_pa(kva, VM_PAGE_TO_PHYS(pgs[j]), VM_PROT_READ); 734 } 735 pmap_update(pmap_kernel()); 736 737 /* Now we can put the pipe in direct write mode */ 738 wpipe->pipe_map.pos = bpos; 739 wpipe->pipe_map.cnt = bcnt; 740 741 /* 742 * But before we can let someone do a direct read, we 743 * have to wait until the pipe is drained. Release the 744 * pipe lock while we wait. 745 */ 746 mutex_enter(lock); 747 wpipe->pipe_state |= PIPE_DIRECTW; 748 pipeunlock(wpipe); 749 750 while (error == 0 && wpipe->pipe_buffer.cnt > 0) { 751 cv_broadcast(&wpipe->pipe_cv); 752 error = cv_wait_sig(&wpipe->pipe_cv, lock); 753 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 754 error = EPIPE; 755 } 756 757 /* Pipe is drained; next read will off the direct buffer */ 758 wpipe->pipe_state |= PIPE_DIRECTR; 759 760 /* Wait until the reader is done */ 761 while (error == 0 && (wpipe->pipe_state & PIPE_DIRECTR)) { 762 cv_broadcast(&wpipe->pipe_cv); 763 pipeselwakeup(wpipe, wpipe, POLL_IN); 764 error = cv_wait_sig(&wpipe->pipe_cv, lock); 765 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 766 error = EPIPE; 767 } 768 769 /* Take pipe out of direct write mode */ 770 wpipe->pipe_state &= ~(PIPE_DIRECTW | PIPE_DIRECTR); 771 772 /* Acquire the pipe lock and cleanup */ 773 (void)pipelock(wpipe, 0); 774 mutex_exit(lock); 775 776 if (pgs != NULL) { 777 pmap_kremove(wpipe->pipe_map.kva, blen); 778 uvm_unloan(pgs, npages, UVM_LOAN_TOPAGE); 779 } 780 if (error || amountpipekva > maxpipekva) 781 pipe_loan_free(wpipe); 782 783 mutex_enter(lock); 784 if (error) { 785 pipeselwakeup(wpipe, wpipe, POLL_ERR); 786 787 /* 788 * If nothing was read from what we offered, return error 789 * straight on. Otherwise update uio resid first. Caller 790 * will deal with the error condition, returning short 791 * write, error, or restarting the write(2) as appropriate. 792 */ 793 if (wpipe->pipe_map.cnt == bcnt) { 794 wpipe->pipe_map.cnt = 0; 795 cv_broadcast(&wpipe->pipe_cv); 796 return (error); 797 } 798 799 bcnt -= wpipe->pipe_map.cnt; 800 } 801 802 uio->uio_resid -= bcnt; 803 /* uio_offset not updated, not set/used for write(2) */ 804 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + bcnt; 805 uio->uio_iov->iov_len -= bcnt; 806 if (uio->uio_iov->iov_len == 0) { 807 uio->uio_iov++; 808 uio->uio_iovcnt--; 809 } 810 811 wpipe->pipe_map.cnt = 0; 812 return (error); 813 } 814 #endif /* !PIPE_NODIRECT */ 815 816 static int 817 pipe_write(struct file *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 818 int flags) 819 { 820 struct pipe *wpipe, *rpipe; 821 struct pipebuf *bp; 822 kmutex_t *lock; 823 int error; 824 825 /* We want to write to our peer */ 826 rpipe = (struct pipe *) fp->f_data; 827 lock = rpipe->pipe_lock; 828 error = 0; 829 830 mutex_enter(lock); 831 wpipe = rpipe->pipe_peer; 832 833 /* 834 * Detect loss of pipe read side, issue SIGPIPE if lost. 835 */ 836 if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF) != 0) { 837 mutex_exit(lock); 838 return EPIPE; 839 } 840 ++wpipe->pipe_busy; 841 842 /* Aquire the long-term pipe lock */ 843 if ((error = pipelock(wpipe, 1)) != 0) { 844 --wpipe->pipe_busy; 845 if (wpipe->pipe_busy == 0) { 846 cv_broadcast(&wpipe->pipe_cv); 847 } 848 mutex_exit(lock); 849 return (error); 850 } 851 852 bp = &wpipe->pipe_buffer; 853 854 /* 855 * If it is advantageous to resize the pipe buffer, do so. 856 */ 857 if ((uio->uio_resid > PIPE_SIZE) && 858 (nbigpipe < maxbigpipes) && 859 #ifndef PIPE_NODIRECT 860 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 861 #endif 862 (bp->size <= PIPE_SIZE) && (bp->cnt == 0)) { 863 864 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) 865 atomic_inc_uint(&nbigpipe); 866 } 867 868 while (uio->uio_resid) { 869 size_t space; 870 871 #ifndef PIPE_NODIRECT 872 /* 873 * Pipe buffered writes cannot be coincidental with 874 * direct writes. Also, only one direct write can be 875 * in progress at any one time. We wait until the currently 876 * executing direct write is completed before continuing. 877 * 878 * We break out if a signal occurs or the reader goes away. 879 */ 880 while (error == 0 && wpipe->pipe_state & PIPE_DIRECTW) { 881 cv_broadcast(&wpipe->pipe_cv); 882 pipeunlock(wpipe); 883 error = cv_wait_sig(&wpipe->pipe_cv, lock); 884 (void)pipelock(wpipe, 0); 885 if (wpipe->pipe_state & PIPE_EOF) 886 error = EPIPE; 887 } 888 if (error) 889 break; 890 891 /* 892 * If the transfer is large, we can gain performance if 893 * we do process-to-process copies directly. 894 * If the write is non-blocking, we don't use the 895 * direct write mechanism. 896 * 897 * The direct write mechanism will detect the reader going 898 * away on us. 899 */ 900 if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) && 901 (fp->f_flag & FNONBLOCK) == 0 && 902 (wpipe->pipe_map.kva || (amountpipekva < limitpipekva))) { 903 error = pipe_direct_write(fp, wpipe, uio); 904 905 /* 906 * Break out if error occurred, unless it's ENOMEM. 907 * ENOMEM means we failed to allocate some resources 908 * for direct write, so we just fallback to ordinary 909 * write. If the direct write was successful, 910 * process rest of data via ordinary write. 911 */ 912 if (error == 0) 913 continue; 914 915 if (error != ENOMEM) 916 break; 917 } 918 #endif /* PIPE_NODIRECT */ 919 920 space = bp->size - bp->cnt; 921 922 /* Writes of size <= PIPE_BUF must be atomic. */ 923 if ((space < uio->uio_resid) && (uio->uio_resid <= PIPE_BUF)) 924 space = 0; 925 926 if (space > 0) { 927 int size; /* Transfer size */ 928 int segsize; /* first segment to transfer */ 929 930 /* 931 * Transfer size is minimum of uio transfer 932 * and free space in pipe buffer. 933 */ 934 if (space > uio->uio_resid) 935 size = uio->uio_resid; 936 else 937 size = space; 938 /* 939 * First segment to transfer is minimum of 940 * transfer size and contiguous space in 941 * pipe buffer. If first segment to transfer 942 * is less than the transfer size, we've got 943 * a wraparound in the buffer. 944 */ 945 segsize = bp->size - bp->in; 946 if (segsize > size) 947 segsize = size; 948 949 /* Transfer first segment */ 950 mutex_exit(lock); 951 error = uiomove((char *)bp->buffer + bp->in, segsize, 952 uio); 953 954 if (error == 0 && segsize < size) { 955 /* 956 * Transfer remaining part now, to 957 * support atomic writes. Wraparound 958 * happened. 959 */ 960 #ifdef DEBUG 961 if (bp->in + segsize != bp->size) 962 panic("Expected pipe buffer wraparound disappeared"); 963 #endif 964 965 error = uiomove(bp->buffer, 966 size - segsize, uio); 967 } 968 mutex_enter(lock); 969 if (error) 970 break; 971 972 bp->in += size; 973 if (bp->in >= bp->size) { 974 #ifdef DEBUG 975 if (bp->in != size - segsize + bp->size) 976 panic("Expected wraparound bad"); 977 #endif 978 bp->in = size - segsize; 979 } 980 981 bp->cnt += size; 982 #ifdef DEBUG 983 if (bp->cnt > bp->size) 984 panic("Pipe buffer overflow"); 985 #endif 986 } else { 987 /* 988 * If the "read-side" has been blocked, wake it up now. 989 */ 990 cv_broadcast(&wpipe->pipe_cv); 991 992 /* 993 * don't block on non-blocking I/O 994 */ 995 if (fp->f_flag & FNONBLOCK) { 996 error = EAGAIN; 997 break; 998 } 999 1000 /* 1001 * We have no more space and have something to offer, 1002 * wake up select/poll. 1003 */ 1004 if (bp->cnt) 1005 pipeselwakeup(wpipe, wpipe, POLL_OUT); 1006 1007 pipeunlock(wpipe); 1008 error = cv_wait_sig(&wpipe->pipe_cv, lock); 1009 (void)pipelock(wpipe, 0); 1010 if (error != 0) 1011 break; 1012 /* 1013 * If read side wants to go away, we just issue a signal 1014 * to ourselves. 1015 */ 1016 if (wpipe->pipe_state & PIPE_EOF) { 1017 error = EPIPE; 1018 break; 1019 } 1020 } 1021 } 1022 1023 --wpipe->pipe_busy; 1024 if (wpipe->pipe_busy == 0 || bp->cnt > 0) { 1025 cv_broadcast(&wpipe->pipe_cv); 1026 } 1027 1028 /* 1029 * Don't return EPIPE if I/O was successful 1030 */ 1031 if (error == EPIPE && bp->cnt == 0 && uio->uio_resid == 0) 1032 error = 0; 1033 1034 if (error == 0) 1035 getmicrotime(&wpipe->pipe_mtime); 1036 1037 /* 1038 * We have something to offer, wake up select/poll. 1039 * wpipe->pipe_map.cnt is always 0 in this point (direct write 1040 * is only done synchronously), so check only wpipe->pipe_buffer.cnt 1041 */ 1042 if (bp->cnt) 1043 pipeselwakeup(wpipe, wpipe, POLL_OUT); 1044 1045 /* 1046 * Arrange for next read(2) to do a signal. 1047 */ 1048 wpipe->pipe_state |= PIPE_SIGNALR; 1049 1050 pipeunlock(wpipe); 1051 mutex_exit(lock); 1052 return (error); 1053 } 1054 1055 /* 1056 * we implement a very minimal set of ioctls for compatibility with sockets. 1057 */ 1058 int 1059 pipe_ioctl(struct file *fp, u_long cmd, void *data, struct lwp *l) 1060 { 1061 struct pipe *pipe = (struct pipe *)fp->f_data; 1062 struct proc *p = l->l_proc; 1063 kmutex_t *lock = pipe->pipe_lock; 1064 1065 switch (cmd) { 1066 1067 case FIONBIO: 1068 return (0); 1069 1070 case FIOASYNC: 1071 mutex_enter(lock); 1072 if (*(int *)data) { 1073 pipe->pipe_state |= PIPE_ASYNC; 1074 } else { 1075 pipe->pipe_state &= ~PIPE_ASYNC; 1076 } 1077 mutex_exit(lock); 1078 return (0); 1079 1080 case FIONREAD: 1081 mutex_enter(lock); 1082 #ifndef PIPE_NODIRECT 1083 if (pipe->pipe_state & PIPE_DIRECTW) 1084 *(int *)data = pipe->pipe_map.cnt; 1085 else 1086 #endif 1087 *(int *)data = pipe->pipe_buffer.cnt; 1088 mutex_exit(lock); 1089 return (0); 1090 1091 case FIONWRITE: 1092 /* Look at other side */ 1093 pipe = pipe->pipe_peer; 1094 mutex_enter(lock); 1095 #ifndef PIPE_NODIRECT 1096 if (pipe->pipe_state & PIPE_DIRECTW) 1097 *(int *)data = pipe->pipe_map.cnt; 1098 else 1099 #endif 1100 *(int *)data = pipe->pipe_buffer.cnt; 1101 mutex_exit(lock); 1102 return (0); 1103 1104 case FIONSPACE: 1105 /* Look at other side */ 1106 pipe = pipe->pipe_peer; 1107 mutex_enter(lock); 1108 #ifndef PIPE_NODIRECT 1109 /* 1110 * If we're in direct-mode, we don't really have a 1111 * send queue, and any other write will block. Thus 1112 * zero seems like the best answer. 1113 */ 1114 if (pipe->pipe_state & PIPE_DIRECTW) 1115 *(int *)data = 0; 1116 else 1117 #endif 1118 *(int *)data = pipe->pipe_buffer.size - 1119 pipe->pipe_buffer.cnt; 1120 mutex_exit(lock); 1121 return (0); 1122 1123 case TIOCSPGRP: 1124 case FIOSETOWN: 1125 return fsetown(p, &pipe->pipe_pgid, cmd, data); 1126 1127 case TIOCGPGRP: 1128 case FIOGETOWN: 1129 return fgetown(p, pipe->pipe_pgid, cmd, data); 1130 1131 } 1132 return (EPASSTHROUGH); 1133 } 1134 1135 int 1136 pipe_poll(struct file *fp, int events, struct lwp *l) 1137 { 1138 struct pipe *rpipe = (struct pipe *)fp->f_data; 1139 struct pipe *wpipe; 1140 int eof = 0; 1141 int revents = 0; 1142 1143 mutex_enter(rpipe->pipe_lock); 1144 wpipe = rpipe->pipe_peer; 1145 1146 if (events & (POLLIN | POLLRDNORM)) 1147 if ((rpipe->pipe_buffer.cnt > 0) || 1148 #ifndef PIPE_NODIRECT 1149 (rpipe->pipe_state & PIPE_DIRECTR) || 1150 #endif 1151 (rpipe->pipe_state & PIPE_EOF)) 1152 revents |= events & (POLLIN | POLLRDNORM); 1153 1154 eof |= (rpipe->pipe_state & PIPE_EOF); 1155 1156 if (wpipe == NULL) 1157 revents |= events & (POLLOUT | POLLWRNORM); 1158 else { 1159 if (events & (POLLOUT | POLLWRNORM)) 1160 if ((wpipe->pipe_state & PIPE_EOF) || ( 1161 #ifndef PIPE_NODIRECT 1162 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 1163 #endif 1164 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 1165 revents |= events & (POLLOUT | POLLWRNORM); 1166 1167 eof |= (wpipe->pipe_state & PIPE_EOF); 1168 } 1169 1170 if (wpipe == NULL || eof) 1171 revents |= POLLHUP; 1172 1173 if (revents == 0) { 1174 if (events & (POLLIN | POLLRDNORM)) 1175 selrecord(l, &rpipe->pipe_sel); 1176 1177 if (events & (POLLOUT | POLLWRNORM)) 1178 selrecord(l, &wpipe->pipe_sel); 1179 } 1180 mutex_exit(rpipe->pipe_lock); 1181 1182 return (revents); 1183 } 1184 1185 static int 1186 pipe_stat(struct file *fp, struct stat *ub, struct lwp *l) 1187 { 1188 struct pipe *pipe = (struct pipe *)fp->f_data; 1189 1190 memset((void *)ub, 0, sizeof(*ub)); 1191 ub->st_mode = S_IFIFO | S_IRUSR | S_IWUSR; 1192 ub->st_blksize = pipe->pipe_buffer.size; 1193 if (ub->st_blksize == 0 && pipe->pipe_peer) 1194 ub->st_blksize = pipe->pipe_peer->pipe_buffer.size; 1195 ub->st_size = pipe->pipe_buffer.cnt; 1196 ub->st_blocks = (ub->st_size) ? 1 : 0; 1197 TIMEVAL_TO_TIMESPEC(&pipe->pipe_atime, &ub->st_atimespec); 1198 TIMEVAL_TO_TIMESPEC(&pipe->pipe_mtime, &ub->st_mtimespec); 1199 TIMEVAL_TO_TIMESPEC(&pipe->pipe_ctime, &ub->st_ctimespec); 1200 ub->st_uid = kauth_cred_geteuid(fp->f_cred); 1201 ub->st_gid = kauth_cred_getegid(fp->f_cred); 1202 1203 /* 1204 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 1205 * XXX (st_dev, st_ino) should be unique. 1206 */ 1207 return (0); 1208 } 1209 1210 /* ARGSUSED */ 1211 static int 1212 pipe_close(struct file *fp, struct lwp *l) 1213 { 1214 struct pipe *pipe = (struct pipe *)fp->f_data; 1215 1216 fp->f_data = NULL; 1217 pipeclose(fp, pipe); 1218 return (0); 1219 } 1220 1221 static void 1222 pipe_free_kmem(struct pipe *pipe) 1223 { 1224 1225 if (pipe->pipe_buffer.buffer != NULL) { 1226 if (pipe->pipe_buffer.size > PIPE_SIZE) 1227 atomic_dec_uint(&nbigpipe); 1228 uvm_km_free(kernel_map, 1229 (vaddr_t)pipe->pipe_buffer.buffer, 1230 pipe->pipe_buffer.size, UVM_KMF_PAGEABLE); 1231 atomic_add_int(&amountpipekva, -pipe->pipe_buffer.size); 1232 pipe->pipe_buffer.buffer = NULL; 1233 } 1234 #ifndef PIPE_NODIRECT 1235 if (pipe->pipe_map.kva != 0) { 1236 pipe_loan_free(pipe); 1237 pipe->pipe_map.cnt = 0; 1238 pipe->pipe_map.kva = 0; 1239 pipe->pipe_map.pos = 0; 1240 pipe->pipe_map.npages = 0; 1241 } 1242 #endif /* !PIPE_NODIRECT */ 1243 } 1244 1245 /* 1246 * shutdown the pipe 1247 */ 1248 static void 1249 pipeclose(struct file *fp, struct pipe *pipe) 1250 { 1251 struct pipe_mutex *mutex; 1252 kmutex_t *lock; 1253 struct pipe *ppipe; 1254 u_int refcnt; 1255 1256 if (pipe == NULL) 1257 return; 1258 lock = pipe->pipe_lock; 1259 mutex_enter(lock); 1260 pipeselwakeup(pipe, pipe, POLL_HUP); 1261 1262 /* 1263 * If the other side is blocked, wake it up saying that 1264 * we want to close it down. 1265 */ 1266 pipe->pipe_state |= PIPE_EOF; 1267 if (pipe->pipe_busy) { 1268 while (pipe->pipe_busy) { 1269 cv_broadcast(&pipe->pipe_cv); 1270 cv_wait_sig(&pipe->pipe_cv, lock); 1271 } 1272 } 1273 1274 /* 1275 * Disconnect from peer 1276 */ 1277 if ((ppipe = pipe->pipe_peer) != NULL) { 1278 pipeselwakeup(ppipe, ppipe, POLL_HUP); 1279 ppipe->pipe_state |= PIPE_EOF; 1280 cv_broadcast(&ppipe->pipe_cv); 1281 ppipe->pipe_peer = NULL; 1282 } 1283 1284 KASSERT((pipe->pipe_state & PIPE_LOCKFL) == 0); 1285 1286 mutex = (struct pipe_mutex *)lock; 1287 refcnt = --(mutex->pm_refcnt); 1288 KASSERT(refcnt == 0 || refcnt == 1); 1289 mutex_exit(lock); 1290 1291 /* 1292 * free resources 1293 */ 1294 pipe_free_kmem(pipe); 1295 cv_destroy(&pipe->pipe_cv); 1296 cv_destroy(&pipe->pipe_lkcv); 1297 seldestroy(&pipe->pipe_sel); 1298 pool_cache_put(pipe_cache, pipe); 1299 if (refcnt == 0) 1300 pool_cache_put(pipe_mutex_cache, mutex); 1301 } 1302 1303 static void 1304 filt_pipedetach(struct knote *kn) 1305 { 1306 struct pipe *pipe; 1307 kmutex_t *lock; 1308 1309 pipe = (struct pipe *)kn->kn_fp->f_data; 1310 lock = pipe->pipe_lock; 1311 1312 mutex_enter(lock); 1313 1314 switch(kn->kn_filter) { 1315 case EVFILT_WRITE: 1316 /* need the peer structure, not our own */ 1317 pipe = pipe->pipe_peer; 1318 1319 /* if reader end already closed, just return */ 1320 if (pipe == NULL) { 1321 mutex_exit(lock); 1322 return; 1323 } 1324 1325 break; 1326 default: 1327 /* nothing to do */ 1328 break; 1329 } 1330 1331 #ifdef DIAGNOSTIC 1332 if (kn->kn_hook != pipe) 1333 panic("filt_pipedetach: inconsistent knote"); 1334 #endif 1335 1336 SLIST_REMOVE(&pipe->pipe_sel.sel_klist, kn, knote, kn_selnext); 1337 mutex_exit(lock); 1338 } 1339 1340 /*ARGSUSED*/ 1341 static int 1342 filt_piperead(struct knote *kn, long hint) 1343 { 1344 struct pipe *rpipe = (struct pipe *)kn->kn_fp->f_data; 1345 struct pipe *wpipe; 1346 1347 if ((hint & NOTE_SUBMIT) == 0) { 1348 mutex_enter(rpipe->pipe_lock); 1349 } 1350 wpipe = rpipe->pipe_peer; 1351 kn->kn_data = rpipe->pipe_buffer.cnt; 1352 1353 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1354 kn->kn_data = rpipe->pipe_map.cnt; 1355 1356 if ((rpipe->pipe_state & PIPE_EOF) || 1357 (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1358 kn->kn_flags |= EV_EOF; 1359 if ((hint & NOTE_SUBMIT) == 0) { 1360 mutex_exit(rpipe->pipe_lock); 1361 } 1362 return (1); 1363 } 1364 1365 if ((hint & NOTE_SUBMIT) == 0) { 1366 mutex_exit(rpipe->pipe_lock); 1367 } 1368 return (kn->kn_data > 0); 1369 } 1370 1371 /*ARGSUSED*/ 1372 static int 1373 filt_pipewrite(struct knote *kn, long hint) 1374 { 1375 struct pipe *rpipe = (struct pipe *)kn->kn_fp->f_data; 1376 struct pipe *wpipe; 1377 1378 if ((hint & NOTE_SUBMIT) == 0) { 1379 mutex_enter(rpipe->pipe_lock); 1380 } 1381 wpipe = rpipe->pipe_peer; 1382 1383 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1384 kn->kn_data = 0; 1385 kn->kn_flags |= EV_EOF; 1386 if ((hint & NOTE_SUBMIT) == 0) { 1387 mutex_exit(rpipe->pipe_lock); 1388 } 1389 return (1); 1390 } 1391 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1392 if (wpipe->pipe_state & PIPE_DIRECTW) 1393 kn->kn_data = 0; 1394 1395 if ((hint & NOTE_SUBMIT) == 0) { 1396 mutex_exit(rpipe->pipe_lock); 1397 } 1398 return (kn->kn_data >= PIPE_BUF); 1399 } 1400 1401 static const struct filterops pipe_rfiltops = 1402 { 1, NULL, filt_pipedetach, filt_piperead }; 1403 static const struct filterops pipe_wfiltops = 1404 { 1, NULL, filt_pipedetach, filt_pipewrite }; 1405 1406 /*ARGSUSED*/ 1407 static int 1408 pipe_kqfilter(struct file *fp, struct knote *kn) 1409 { 1410 struct pipe *pipe; 1411 kmutex_t *lock; 1412 1413 pipe = (struct pipe *)kn->kn_fp->f_data; 1414 lock = pipe->pipe_lock; 1415 1416 mutex_enter(lock); 1417 1418 switch (kn->kn_filter) { 1419 case EVFILT_READ: 1420 kn->kn_fop = &pipe_rfiltops; 1421 break; 1422 case EVFILT_WRITE: 1423 kn->kn_fop = &pipe_wfiltops; 1424 pipe = pipe->pipe_peer; 1425 if (pipe == NULL) { 1426 /* other end of pipe has been closed */ 1427 mutex_exit(lock); 1428 return (EBADF); 1429 } 1430 break; 1431 default: 1432 mutex_exit(lock); 1433 return (EINVAL); 1434 } 1435 1436 kn->kn_hook = pipe; 1437 SLIST_INSERT_HEAD(&pipe->pipe_sel.sel_klist, kn, kn_selnext); 1438 mutex_exit(lock); 1439 1440 return (0); 1441 } 1442 1443 /* 1444 * Handle pipe sysctls. 1445 */ 1446 SYSCTL_SETUP(sysctl_kern_pipe_setup, "sysctl kern.pipe subtree setup") 1447 { 1448 1449 sysctl_createv(clog, 0, NULL, NULL, 1450 CTLFLAG_PERMANENT, 1451 CTLTYPE_NODE, "kern", NULL, 1452 NULL, 0, NULL, 0, 1453 CTL_KERN, CTL_EOL); 1454 sysctl_createv(clog, 0, NULL, NULL, 1455 CTLFLAG_PERMANENT, 1456 CTLTYPE_NODE, "pipe", 1457 SYSCTL_DESCR("Pipe settings"), 1458 NULL, 0, NULL, 0, 1459 CTL_KERN, KERN_PIPE, CTL_EOL); 1460 1461 sysctl_createv(clog, 0, NULL, NULL, 1462 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1463 CTLTYPE_INT, "maxkvasz", 1464 SYSCTL_DESCR("Maximum amount of kernel memory to be " 1465 "used for pipes"), 1466 NULL, 0, &maxpipekva, 0, 1467 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXKVASZ, CTL_EOL); 1468 sysctl_createv(clog, 0, NULL, NULL, 1469 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1470 CTLTYPE_INT, "maxloankvasz", 1471 SYSCTL_DESCR("Limit for direct transfers via page loan"), 1472 NULL, 0, &limitpipekva, 0, 1473 CTL_KERN, KERN_PIPE, KERN_PIPE_LIMITKVA, CTL_EOL); 1474 sysctl_createv(clog, 0, NULL, NULL, 1475 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1476 CTLTYPE_INT, "maxbigpipes", 1477 SYSCTL_DESCR("Maximum number of \"big\" pipes"), 1478 NULL, 0, &maxbigpipes, 0, 1479 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXBIGPIPES, CTL_EOL); 1480 sysctl_createv(clog, 0, NULL, NULL, 1481 CTLFLAG_PERMANENT, 1482 CTLTYPE_INT, "nbigpipes", 1483 SYSCTL_DESCR("Number of \"big\" pipes"), 1484 NULL, 0, &nbigpipe, 0, 1485 CTL_KERN, KERN_PIPE, KERN_PIPE_NBIGPIPES, CTL_EOL); 1486 sysctl_createv(clog, 0, NULL, NULL, 1487 CTLFLAG_PERMANENT, 1488 CTLTYPE_INT, "kvasize", 1489 SYSCTL_DESCR("Amount of kernel memory consumed by pipe " 1490 "buffers"), 1491 NULL, 0, &amountpipekva, 0, 1492 CTL_KERN, KERN_PIPE, KERN_PIPE_KVASIZE, CTL_EOL); 1493 } 1494