1 /* $NetBSD: sys_pipe.c,v 1.100 2008/03/27 18:30:15 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2003, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Paul Kranenburg, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Copyright (c) 1996 John S. Dyson 41 * All rights reserved. 42 * 43 * Redistribution and use in source and binary forms, with or without 44 * modification, are permitted provided that the following conditions 45 * are met: 46 * 1. Redistributions of source code must retain the above copyright 47 * notice immediately at the beginning of the file, without modification, 48 * this list of conditions, and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. Absolutely no warranty of function or purpose is made by the author 53 * John S. Dyson. 54 * 4. Modifications may be freely made to this file if the above conditions 55 * are met. 56 * 57 * $FreeBSD: src/sys/kern/sys_pipe.c,v 1.95 2002/03/09 22:06:31 alfred Exp $ 58 */ 59 60 /* 61 * This file contains a high-performance replacement for the socket-based 62 * pipes scheme originally used in FreeBSD/4.4Lite. It does not support 63 * all features of sockets, but does do everything that pipes normally 64 * do. 65 * 66 * Adaption for NetBSD UVM, including uvm_loan() based direct write, was 67 * written by Jaromir Dolecek. 68 */ 69 70 /* 71 * This code has two modes of operation, a small write mode and a large 72 * write mode. The small write mode acts like conventional pipes with 73 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 74 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 75 * and PIPE_SIZE in size it is mapped read-only into the kernel address space 76 * using the UVM page loan facility from where the receiving process can copy 77 * the data directly from the pages in the sending process. 78 * 79 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 80 * happen for small transfers so that the system will not spend all of 81 * its time context switching. PIPE_SIZE is constrained by the 82 * amount of kernel virtual memory. 83 */ 84 85 #include <sys/cdefs.h> 86 __KERNEL_RCSID(0, "$NetBSD: sys_pipe.c,v 1.100 2008/03/27 18:30:15 ad Exp $"); 87 88 #include <sys/param.h> 89 #include <sys/systm.h> 90 #include <sys/proc.h> 91 #include <sys/fcntl.h> 92 #include <sys/file.h> 93 #include <sys/filedesc.h> 94 #include <sys/filio.h> 95 #include <sys/kernel.h> 96 #include <sys/ttycom.h> 97 #include <sys/stat.h> 98 #include <sys/malloc.h> 99 #include <sys/poll.h> 100 #include <sys/signalvar.h> 101 #include <sys/vnode.h> 102 #include <sys/uio.h> 103 #include <sys/select.h> 104 #include <sys/mount.h> 105 #include <sys/syscallargs.h> 106 #include <sys/sysctl.h> 107 #include <sys/kauth.h> 108 #include <sys/atomic.h> 109 #include <sys/pipe.h> 110 111 #include <uvm/uvm.h> 112 113 /* 114 * Use this define if you want to disable *fancy* VM things. Expect an 115 * approx 30% decrease in transfer rate. 116 */ 117 /* #define PIPE_NODIRECT */ 118 119 /* 120 * interfaces to the outside world 121 */ 122 static int pipe_read(struct file *fp, off_t *offset, struct uio *uio, 123 kauth_cred_t cred, int flags); 124 static int pipe_write(struct file *fp, off_t *offset, struct uio *uio, 125 kauth_cred_t cred, int flags); 126 static int pipe_close(struct file *fp); 127 static int pipe_poll(struct file *fp, int events); 128 static int pipe_kqfilter(struct file *fp, struct knote *kn); 129 static int pipe_stat(struct file *fp, struct stat *sb); 130 static int pipe_ioctl(struct file *fp, u_long cmd, void *data); 131 132 static const struct fileops pipeops = { 133 pipe_read, pipe_write, pipe_ioctl, fnullop_fcntl, pipe_poll, 134 pipe_stat, pipe_close, pipe_kqfilter 135 }; 136 137 /* 138 * Single mutex shared between both ends of the pipe. 139 */ 140 141 struct pipe_mutex { 142 kmutex_t pm_mutex; 143 u_int pm_refcnt; 144 }; 145 146 /* 147 * Default pipe buffer size(s), this can be kind-of large now because pipe 148 * space is pageable. The pipe code will try to maintain locality of 149 * reference for performance reasons, so small amounts of outstanding I/O 150 * will not wipe the cache. 151 */ 152 #define MINPIPESIZE (PIPE_SIZE/3) 153 #define MAXPIPESIZE (2*PIPE_SIZE/3) 154 155 /* 156 * Maximum amount of kva for pipes -- this is kind-of a soft limit, but 157 * is there so that on large systems, we don't exhaust it. 158 */ 159 #define MAXPIPEKVA (8*1024*1024) 160 static u_int maxpipekva = MAXPIPEKVA; 161 162 /* 163 * Limit for direct transfers, we cannot, of course limit 164 * the amount of kva for pipes in general though. 165 */ 166 #define LIMITPIPEKVA (16*1024*1024) 167 static u_int limitpipekva = LIMITPIPEKVA; 168 169 /* 170 * Limit the number of "big" pipes 171 */ 172 #define LIMITBIGPIPES 32 173 static u_int maxbigpipes = LIMITBIGPIPES; 174 static u_int nbigpipe = 0; 175 176 /* 177 * Amount of KVA consumed by pipe buffers. 178 */ 179 static u_int amountpipekva = 0; 180 181 MALLOC_DEFINE(M_PIPE, "pipe", "Pipe structures"); 182 183 static void pipeclose(struct file *fp, struct pipe *pipe); 184 static void pipe_free_kmem(struct pipe *pipe); 185 static int pipe_create(struct pipe **pipep, int allockva, struct pipe_mutex *); 186 static int pipelock(struct pipe *pipe, int catch); 187 static inline void pipeunlock(struct pipe *pipe); 188 static void pipeselwakeup(struct pipe *pipe, struct pipe *sigp, int code); 189 #ifndef PIPE_NODIRECT 190 static int pipe_direct_write(struct file *fp, struct pipe *wpipe, 191 struct uio *uio); 192 #endif 193 static int pipespace(struct pipe *pipe, int size); 194 195 #ifndef PIPE_NODIRECT 196 static int pipe_loan_alloc(struct pipe *, int); 197 static void pipe_loan_free(struct pipe *); 198 #endif /* PIPE_NODIRECT */ 199 200 static int pipe_mutex_ctor(void *, void *, int); 201 static void pipe_mutex_dtor(void *, void *); 202 203 static pool_cache_t pipe_cache; 204 static pool_cache_t pipe_mutex_cache; 205 206 void 207 pipe_init(void) 208 { 209 210 pipe_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "pipepl", 211 NULL, IPL_NONE, NULL, NULL, NULL); 212 KASSERT(pipe_cache != NULL); 213 214 pipe_mutex_cache = pool_cache_init(sizeof(struct pipe_mutex), 215 coherency_unit, 0, 0, "pipemtxpl", NULL, IPL_NONE, pipe_mutex_ctor, 216 pipe_mutex_dtor, NULL); 217 KASSERT(pipe_cache != NULL); 218 } 219 220 static int 221 pipe_mutex_ctor(void *arg, void *obj, int flag) 222 { 223 struct pipe_mutex *pm = obj; 224 225 mutex_init(&pm->pm_mutex, MUTEX_DEFAULT, IPL_NONE); 226 pm->pm_refcnt = 0; 227 228 return 0; 229 } 230 231 static void 232 pipe_mutex_dtor(void *arg, void *obj) 233 { 234 struct pipe_mutex *pm = obj; 235 236 KASSERT(pm->pm_refcnt == 0); 237 238 mutex_destroy(&pm->pm_mutex); 239 } 240 241 /* 242 * The pipe system call for the DTYPE_PIPE type of pipes 243 */ 244 245 /* ARGSUSED */ 246 int 247 sys_pipe(struct lwp *l, const void *v, register_t *retval) 248 { 249 struct file *rf, *wf; 250 struct pipe *rpipe, *wpipe; 251 struct pipe_mutex *mutex; 252 int fd, error; 253 proc_t *p; 254 255 p = curproc; 256 rpipe = wpipe = NULL; 257 mutex = pool_cache_get(pipe_mutex_cache, PR_WAITOK); 258 if (mutex == NULL) 259 return (ENOMEM); 260 if (pipe_create(&rpipe, 1, mutex) || pipe_create(&wpipe, 0, mutex)) { 261 pipeclose(NULL, rpipe); 262 pipeclose(NULL, wpipe); 263 return (ENFILE); 264 } 265 266 error = fd_allocfile(&rf, &fd); 267 if (error) 268 goto free2; 269 retval[0] = fd; 270 rf->f_flag = FREAD; 271 rf->f_type = DTYPE_PIPE; 272 rf->f_data = (void *)rpipe; 273 rf->f_ops = &pipeops; 274 275 error = fd_allocfile(&wf, &fd); 276 if (error) 277 goto free3; 278 retval[1] = fd; 279 wf->f_flag = FWRITE; 280 wf->f_type = DTYPE_PIPE; 281 wf->f_data = (void *)wpipe; 282 wf->f_ops = &pipeops; 283 284 rpipe->pipe_peer = wpipe; 285 wpipe->pipe_peer = rpipe; 286 287 fd_affix(p, rf, (int)retval[0]); 288 fd_affix(p, wf, (int)retval[1]); 289 return (0); 290 free3: 291 fd_abort(p, rf, (int)retval[0]); 292 free2: 293 pipeclose(NULL, wpipe); 294 pipeclose(NULL, rpipe); 295 296 return (error); 297 } 298 299 /* 300 * Allocate kva for pipe circular buffer, the space is pageable 301 * This routine will 'realloc' the size of a pipe safely, if it fails 302 * it will retain the old buffer. 303 * If it fails it will return ENOMEM. 304 */ 305 static int 306 pipespace(struct pipe *pipe, int size) 307 { 308 void *buffer; 309 /* 310 * Allocate pageable virtual address space. Physical memory is 311 * allocated on demand. 312 */ 313 buffer = (void *) uvm_km_alloc(kernel_map, round_page(size), 0, 314 UVM_KMF_PAGEABLE); 315 if (buffer == NULL) 316 return (ENOMEM); 317 318 /* free old resources if we're resizing */ 319 pipe_free_kmem(pipe); 320 pipe->pipe_buffer.buffer = buffer; 321 pipe->pipe_buffer.size = size; 322 pipe->pipe_buffer.in = 0; 323 pipe->pipe_buffer.out = 0; 324 pipe->pipe_buffer.cnt = 0; 325 atomic_add_int(&amountpipekva, pipe->pipe_buffer.size); 326 return (0); 327 } 328 329 /* 330 * Initialize and allocate VM and memory for pipe. 331 */ 332 static int 333 pipe_create(struct pipe **pipep, int allockva, struct pipe_mutex *mutex) 334 { 335 struct pipe *pipe; 336 int error; 337 338 pipe = *pipep = pool_cache_get(pipe_cache, PR_WAITOK); 339 mutex->pm_refcnt++; 340 341 /* Initialize */ 342 memset(pipe, 0, sizeof(struct pipe)); 343 pipe->pipe_state = PIPE_SIGNALR; 344 345 getmicrotime(&pipe->pipe_ctime); 346 pipe->pipe_atime = pipe->pipe_ctime; 347 pipe->pipe_mtime = pipe->pipe_ctime; 348 pipe->pipe_lock = &mutex->pm_mutex; 349 cv_init(&pipe->pipe_rcv, "piperd"); 350 cv_init(&pipe->pipe_wcv, "pipewr"); 351 cv_init(&pipe->pipe_draincv, "pipedrain"); 352 cv_init(&pipe->pipe_lkcv, "pipelk"); 353 selinit(&pipe->pipe_sel); 354 355 if (allockva && (error = pipespace(pipe, PIPE_SIZE))) 356 return (error); 357 358 return (0); 359 } 360 361 362 /* 363 * Lock a pipe for I/O, blocking other access 364 * Called with pipe spin lock held. 365 * Return with pipe spin lock released on success. 366 */ 367 static int 368 pipelock(struct pipe *pipe, int catch) 369 { 370 int error; 371 372 KASSERT(mutex_owned(pipe->pipe_lock)); 373 374 while (pipe->pipe_state & PIPE_LOCKFL) { 375 pipe->pipe_state |= PIPE_LWANT; 376 if (catch) { 377 error = cv_wait_sig(&pipe->pipe_lkcv, pipe->pipe_lock); 378 if (error != 0) 379 return error; 380 } else 381 cv_wait(&pipe->pipe_lkcv, pipe->pipe_lock); 382 } 383 384 pipe->pipe_state |= PIPE_LOCKFL; 385 386 return 0; 387 } 388 389 /* 390 * unlock a pipe I/O lock 391 */ 392 static inline void 393 pipeunlock(struct pipe *pipe) 394 { 395 396 KASSERT(pipe->pipe_state & PIPE_LOCKFL); 397 398 pipe->pipe_state &= ~PIPE_LOCKFL; 399 if (pipe->pipe_state & PIPE_LWANT) { 400 pipe->pipe_state &= ~PIPE_LWANT; 401 cv_broadcast(&pipe->pipe_lkcv); 402 } 403 } 404 405 /* 406 * Select/poll wakup. This also sends SIGIO to peer connected to 407 * 'sigpipe' side of pipe. 408 */ 409 static void 410 pipeselwakeup(struct pipe *selp, struct pipe *sigp, int code) 411 { 412 int band; 413 414 switch (code) { 415 case POLL_IN: 416 band = POLLIN|POLLRDNORM; 417 break; 418 case POLL_OUT: 419 band = POLLOUT|POLLWRNORM; 420 break; 421 case POLL_HUP: 422 band = POLLHUP; 423 break; 424 #if POLL_HUP != POLL_ERR 425 case POLL_ERR: 426 band = POLLERR; 427 break; 428 #endif 429 default: 430 band = 0; 431 #ifdef DIAGNOSTIC 432 printf("bad siginfo code %d in pipe notification.\n", code); 433 #endif 434 break; 435 } 436 437 selnotify(&selp->pipe_sel, band, NOTE_SUBMIT); 438 439 if (sigp == NULL || (sigp->pipe_state & PIPE_ASYNC) == 0) 440 return; 441 442 fownsignal(sigp->pipe_pgid, SIGIO, code, band, selp); 443 } 444 445 /* ARGSUSED */ 446 static int 447 pipe_read(struct file *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 448 int flags) 449 { 450 struct pipe *rpipe = (struct pipe *) fp->f_data; 451 struct pipebuf *bp = &rpipe->pipe_buffer; 452 kmutex_t *lock = rpipe->pipe_lock; 453 int error; 454 size_t nread = 0; 455 size_t size; 456 size_t ocnt; 457 458 mutex_enter(lock); 459 ++rpipe->pipe_busy; 460 ocnt = bp->cnt; 461 462 again: 463 error = pipelock(rpipe, 1); 464 if (error) 465 goto unlocked_error; 466 467 while (uio->uio_resid) { 468 /* 469 * normal pipe buffer receive 470 */ 471 if (bp->cnt > 0) { 472 size = bp->size - bp->out; 473 if (size > bp->cnt) 474 size = bp->cnt; 475 if (size > uio->uio_resid) 476 size = uio->uio_resid; 477 478 mutex_exit(lock); 479 error = uiomove((char *)bp->buffer + bp->out, size, uio); 480 mutex_enter(lock); 481 if (error) 482 break; 483 484 bp->out += size; 485 if (bp->out >= bp->size) 486 bp->out = 0; 487 488 bp->cnt -= size; 489 490 /* 491 * If there is no more to read in the pipe, reset 492 * its pointers to the beginning. This improves 493 * cache hit stats. 494 */ 495 if (bp->cnt == 0) { 496 bp->in = 0; 497 bp->out = 0; 498 } 499 nread += size; 500 continue; 501 } 502 503 #ifndef PIPE_NODIRECT 504 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) { 505 /* 506 * Direct copy, bypassing a kernel buffer. 507 */ 508 void * va; 509 510 KASSERT(rpipe->pipe_state & PIPE_DIRECTW); 511 512 size = rpipe->pipe_map.cnt; 513 if (size > uio->uio_resid) 514 size = uio->uio_resid; 515 516 va = (char *)rpipe->pipe_map.kva + rpipe->pipe_map.pos; 517 mutex_exit(lock); 518 error = uiomove(va, size, uio); 519 mutex_enter(lock); 520 if (error) 521 break; 522 nread += size; 523 rpipe->pipe_map.pos += size; 524 rpipe->pipe_map.cnt -= size; 525 if (rpipe->pipe_map.cnt == 0) { 526 rpipe->pipe_state &= ~PIPE_DIRECTR; 527 cv_broadcast(&rpipe->pipe_wcv); 528 } 529 continue; 530 } 531 #endif 532 /* 533 * Break if some data was read. 534 */ 535 if (nread > 0) 536 break; 537 538 /* 539 * detect EOF condition 540 * read returns 0 on EOF, no need to set error 541 */ 542 if (rpipe->pipe_state & PIPE_EOF) 543 break; 544 545 /* 546 * don't block on non-blocking I/O 547 */ 548 if (fp->f_flag & FNONBLOCK) { 549 error = EAGAIN; 550 break; 551 } 552 553 /* 554 * Unlock the pipe buffer for our remaining processing. 555 * We will either break out with an error or we will 556 * sleep and relock to loop. 557 */ 558 pipeunlock(rpipe); 559 560 /* 561 * Re-check to see if more direct writes are pending. 562 */ 563 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) 564 goto again; 565 566 /* 567 * We want to read more, wake up select/poll. 568 */ 569 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_IN); 570 571 /* 572 * If the "write-side" is blocked, wake it up now. 573 */ 574 cv_broadcast(&rpipe->pipe_wcv); 575 576 /* Now wait until the pipe is filled */ 577 error = cv_wait_sig(&rpipe->pipe_rcv, lock); 578 if (error != 0) 579 goto unlocked_error; 580 goto again; 581 } 582 583 if (error == 0) 584 getmicrotime(&rpipe->pipe_atime); 585 pipeunlock(rpipe); 586 587 unlocked_error: 588 --rpipe->pipe_busy; 589 if (rpipe->pipe_busy == 0) { 590 cv_broadcast(&rpipe->pipe_draincv); 591 } 592 if (bp->cnt < MINPIPESIZE) { 593 cv_broadcast(&rpipe->pipe_wcv); 594 } 595 596 /* 597 * If anything was read off the buffer, signal to the writer it's 598 * possible to write more data. Also send signal if we are here for the 599 * first time after last write. 600 */ 601 if ((bp->size - bp->cnt) >= PIPE_BUF 602 && (ocnt != bp->cnt || (rpipe->pipe_state & PIPE_SIGNALR))) { 603 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 604 rpipe->pipe_state &= ~PIPE_SIGNALR; 605 } 606 607 mutex_exit(lock); 608 return (error); 609 } 610 611 #ifndef PIPE_NODIRECT 612 /* 613 * Allocate structure for loan transfer. 614 */ 615 static int 616 pipe_loan_alloc(struct pipe *wpipe, int npages) 617 { 618 vsize_t len; 619 620 len = (vsize_t)npages << PAGE_SHIFT; 621 atomic_add_int(&amountpipekva, len); 622 wpipe->pipe_map.kva = uvm_km_alloc(kernel_map, len, 0, 623 UVM_KMF_VAONLY | UVM_KMF_WAITVA); 624 if (wpipe->pipe_map.kva == 0) { 625 atomic_add_int(&amountpipekva, -len); 626 return (ENOMEM); 627 } 628 629 wpipe->pipe_map.npages = npages; 630 wpipe->pipe_map.pgs = malloc(npages * sizeof(struct vm_page *), M_PIPE, 631 M_WAITOK); 632 return (0); 633 } 634 635 /* 636 * Free resources allocated for loan transfer. 637 */ 638 static void 639 pipe_loan_free(struct pipe *wpipe) 640 { 641 vsize_t len; 642 643 len = (vsize_t)wpipe->pipe_map.npages << PAGE_SHIFT; 644 uvm_km_free(kernel_map, wpipe->pipe_map.kva, len, UVM_KMF_VAONLY); 645 wpipe->pipe_map.kva = 0; 646 atomic_add_int(&amountpipekva, -len); 647 free(wpipe->pipe_map.pgs, M_PIPE); 648 wpipe->pipe_map.pgs = NULL; 649 } 650 651 /* 652 * NetBSD direct write, using uvm_loan() mechanism. 653 * This implements the pipe buffer write mechanism. Note that only 654 * a direct write OR a normal pipe write can be pending at any given time. 655 * If there are any characters in the pipe buffer, the direct write will 656 * be deferred until the receiving process grabs all of the bytes from 657 * the pipe buffer. Then the direct mapping write is set-up. 658 * 659 * Called with the long-term pipe lock held. 660 */ 661 static int 662 pipe_direct_write(struct file *fp, struct pipe *wpipe, struct uio *uio) 663 { 664 int error, npages, j; 665 struct vm_page **pgs; 666 vaddr_t bbase, kva, base, bend; 667 vsize_t blen, bcnt; 668 voff_t bpos; 669 kmutex_t *lock = wpipe->pipe_lock; 670 671 KASSERT(mutex_owned(wpipe->pipe_lock)); 672 KASSERT(wpipe->pipe_map.cnt == 0); 673 674 mutex_exit(lock); 675 676 /* 677 * Handle first PIPE_CHUNK_SIZE bytes of buffer. Deal with buffers 678 * not aligned to PAGE_SIZE. 679 */ 680 bbase = (vaddr_t)uio->uio_iov->iov_base; 681 base = trunc_page(bbase); 682 bend = round_page(bbase + uio->uio_iov->iov_len); 683 blen = bend - base; 684 bpos = bbase - base; 685 686 if (blen > PIPE_DIRECT_CHUNK) { 687 blen = PIPE_DIRECT_CHUNK; 688 bend = base + blen; 689 bcnt = PIPE_DIRECT_CHUNK - bpos; 690 } else { 691 bcnt = uio->uio_iov->iov_len; 692 } 693 npages = blen >> PAGE_SHIFT; 694 695 /* 696 * Free the old kva if we need more pages than we have 697 * allocated. 698 */ 699 if (wpipe->pipe_map.kva != 0 && npages > wpipe->pipe_map.npages) 700 pipe_loan_free(wpipe); 701 702 /* Allocate new kva. */ 703 if (wpipe->pipe_map.kva == 0) { 704 error = pipe_loan_alloc(wpipe, npages); 705 if (error) { 706 mutex_enter(lock); 707 return (error); 708 } 709 } 710 711 /* Loan the write buffer memory from writer process */ 712 pgs = wpipe->pipe_map.pgs; 713 error = uvm_loan(&uio->uio_vmspace->vm_map, base, blen, 714 pgs, UVM_LOAN_TOPAGE); 715 if (error) { 716 pipe_loan_free(wpipe); 717 mutex_enter(lock); 718 return (ENOMEM); /* so that caller fallback to ordinary write */ 719 } 720 721 /* Enter the loaned pages to kva */ 722 kva = wpipe->pipe_map.kva; 723 for (j = 0; j < npages; j++, kva += PAGE_SIZE) { 724 pmap_kenter_pa(kva, VM_PAGE_TO_PHYS(pgs[j]), VM_PROT_READ); 725 } 726 pmap_update(pmap_kernel()); 727 728 /* Now we can put the pipe in direct write mode */ 729 wpipe->pipe_map.pos = bpos; 730 wpipe->pipe_map.cnt = bcnt; 731 732 /* 733 * But before we can let someone do a direct read, we 734 * have to wait until the pipe is drained. Release the 735 * pipe lock while we wait. 736 */ 737 mutex_enter(lock); 738 wpipe->pipe_state |= PIPE_DIRECTW; 739 pipeunlock(wpipe); 740 741 while (error == 0 && wpipe->pipe_buffer.cnt > 0) { 742 cv_broadcast(&wpipe->pipe_rcv); 743 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 744 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 745 error = EPIPE; 746 } 747 748 /* Pipe is drained; next read will off the direct buffer */ 749 wpipe->pipe_state |= PIPE_DIRECTR; 750 751 /* Wait until the reader is done */ 752 while (error == 0 && (wpipe->pipe_state & PIPE_DIRECTR)) { 753 cv_broadcast(&wpipe->pipe_rcv); 754 pipeselwakeup(wpipe, wpipe, POLL_IN); 755 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 756 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 757 error = EPIPE; 758 } 759 760 /* Take pipe out of direct write mode */ 761 wpipe->pipe_state &= ~(PIPE_DIRECTW | PIPE_DIRECTR); 762 763 /* Acquire the pipe lock and cleanup */ 764 (void)pipelock(wpipe, 0); 765 mutex_exit(lock); 766 767 if (pgs != NULL) { 768 pmap_kremove(wpipe->pipe_map.kva, blen); 769 pmap_update(pmap_kernel()); 770 uvm_unloan(pgs, npages, UVM_LOAN_TOPAGE); 771 } 772 if (error || amountpipekva > maxpipekva) 773 pipe_loan_free(wpipe); 774 775 mutex_enter(lock); 776 if (error) { 777 pipeselwakeup(wpipe, wpipe, POLL_ERR); 778 779 /* 780 * If nothing was read from what we offered, return error 781 * straight on. Otherwise update uio resid first. Caller 782 * will deal with the error condition, returning short 783 * write, error, or restarting the write(2) as appropriate. 784 */ 785 if (wpipe->pipe_map.cnt == bcnt) { 786 wpipe->pipe_map.cnt = 0; 787 cv_broadcast(&wpipe->pipe_wcv); 788 return (error); 789 } 790 791 bcnt -= wpipe->pipe_map.cnt; 792 } 793 794 uio->uio_resid -= bcnt; 795 /* uio_offset not updated, not set/used for write(2) */ 796 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + bcnt; 797 uio->uio_iov->iov_len -= bcnt; 798 if (uio->uio_iov->iov_len == 0) { 799 uio->uio_iov++; 800 uio->uio_iovcnt--; 801 } 802 803 wpipe->pipe_map.cnt = 0; 804 return (error); 805 } 806 #endif /* !PIPE_NODIRECT */ 807 808 static int 809 pipe_write(struct file *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 810 int flags) 811 { 812 struct pipe *wpipe, *rpipe; 813 struct pipebuf *bp; 814 kmutex_t *lock; 815 int error; 816 817 /* We want to write to our peer */ 818 rpipe = (struct pipe *) fp->f_data; 819 lock = rpipe->pipe_lock; 820 error = 0; 821 822 mutex_enter(lock); 823 wpipe = rpipe->pipe_peer; 824 825 /* 826 * Detect loss of pipe read side, issue SIGPIPE if lost. 827 */ 828 if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF) != 0) { 829 mutex_exit(lock); 830 return EPIPE; 831 } 832 ++wpipe->pipe_busy; 833 834 /* Aquire the long-term pipe lock */ 835 if ((error = pipelock(wpipe, 1)) != 0) { 836 --wpipe->pipe_busy; 837 if (wpipe->pipe_busy == 0) { 838 cv_broadcast(&wpipe->pipe_draincv); 839 } 840 mutex_exit(lock); 841 return (error); 842 } 843 844 bp = &wpipe->pipe_buffer; 845 846 /* 847 * If it is advantageous to resize the pipe buffer, do so. 848 */ 849 if ((uio->uio_resid > PIPE_SIZE) && 850 (nbigpipe < maxbigpipes) && 851 #ifndef PIPE_NODIRECT 852 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 853 #endif 854 (bp->size <= PIPE_SIZE) && (bp->cnt == 0)) { 855 856 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) 857 atomic_inc_uint(&nbigpipe); 858 } 859 860 while (uio->uio_resid) { 861 size_t space; 862 863 #ifndef PIPE_NODIRECT 864 /* 865 * Pipe buffered writes cannot be coincidental with 866 * direct writes. Also, only one direct write can be 867 * in progress at any one time. We wait until the currently 868 * executing direct write is completed before continuing. 869 * 870 * We break out if a signal occurs or the reader goes away. 871 */ 872 while (error == 0 && wpipe->pipe_state & PIPE_DIRECTW) { 873 cv_broadcast(&wpipe->pipe_rcv); 874 pipeunlock(wpipe); 875 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 876 (void)pipelock(wpipe, 0); 877 if (wpipe->pipe_state & PIPE_EOF) 878 error = EPIPE; 879 } 880 if (error) 881 break; 882 883 /* 884 * If the transfer is large, we can gain performance if 885 * we do process-to-process copies directly. 886 * If the write is non-blocking, we don't use the 887 * direct write mechanism. 888 * 889 * The direct write mechanism will detect the reader going 890 * away on us. 891 */ 892 if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) && 893 (fp->f_flag & FNONBLOCK) == 0 && 894 (wpipe->pipe_map.kva || (amountpipekva < limitpipekva))) { 895 error = pipe_direct_write(fp, wpipe, uio); 896 897 /* 898 * Break out if error occurred, unless it's ENOMEM. 899 * ENOMEM means we failed to allocate some resources 900 * for direct write, so we just fallback to ordinary 901 * write. If the direct write was successful, 902 * process rest of data via ordinary write. 903 */ 904 if (error == 0) 905 continue; 906 907 if (error != ENOMEM) 908 break; 909 } 910 #endif /* PIPE_NODIRECT */ 911 912 space = bp->size - bp->cnt; 913 914 /* Writes of size <= PIPE_BUF must be atomic. */ 915 if ((space < uio->uio_resid) && (uio->uio_resid <= PIPE_BUF)) 916 space = 0; 917 918 if (space > 0) { 919 int size; /* Transfer size */ 920 int segsize; /* first segment to transfer */ 921 922 /* 923 * Transfer size is minimum of uio transfer 924 * and free space in pipe buffer. 925 */ 926 if (space > uio->uio_resid) 927 size = uio->uio_resid; 928 else 929 size = space; 930 /* 931 * First segment to transfer is minimum of 932 * transfer size and contiguous space in 933 * pipe buffer. If first segment to transfer 934 * is less than the transfer size, we've got 935 * a wraparound in the buffer. 936 */ 937 segsize = bp->size - bp->in; 938 if (segsize > size) 939 segsize = size; 940 941 /* Transfer first segment */ 942 mutex_exit(lock); 943 error = uiomove((char *)bp->buffer + bp->in, segsize, 944 uio); 945 946 if (error == 0 && segsize < size) { 947 /* 948 * Transfer remaining part now, to 949 * support atomic writes. Wraparound 950 * happened. 951 */ 952 #ifdef DEBUG 953 if (bp->in + segsize != bp->size) 954 panic("Expected pipe buffer wraparound disappeared"); 955 #endif 956 957 error = uiomove(bp->buffer, 958 size - segsize, uio); 959 } 960 mutex_enter(lock); 961 if (error) 962 break; 963 964 bp->in += size; 965 if (bp->in >= bp->size) { 966 #ifdef DEBUG 967 if (bp->in != size - segsize + bp->size) 968 panic("Expected wraparound bad"); 969 #endif 970 bp->in = size - segsize; 971 } 972 973 bp->cnt += size; 974 #ifdef DEBUG 975 if (bp->cnt > bp->size) 976 panic("Pipe buffer overflow"); 977 #endif 978 } else { 979 /* 980 * If the "read-side" has been blocked, wake it up now. 981 */ 982 cv_broadcast(&wpipe->pipe_rcv); 983 984 /* 985 * don't block on non-blocking I/O 986 */ 987 if (fp->f_flag & FNONBLOCK) { 988 error = EAGAIN; 989 break; 990 } 991 992 /* 993 * We have no more space and have something to offer, 994 * wake up select/poll. 995 */ 996 if (bp->cnt) 997 pipeselwakeup(wpipe, wpipe, POLL_OUT); 998 999 pipeunlock(wpipe); 1000 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 1001 (void)pipelock(wpipe, 0); 1002 if (error != 0) 1003 break; 1004 /* 1005 * If read side wants to go away, we just issue a signal 1006 * to ourselves. 1007 */ 1008 if (wpipe->pipe_state & PIPE_EOF) { 1009 error = EPIPE; 1010 break; 1011 } 1012 } 1013 } 1014 1015 --wpipe->pipe_busy; 1016 if (wpipe->pipe_busy == 0) { 1017 cv_broadcast(&wpipe->pipe_draincv); 1018 } 1019 if (bp->cnt > 0) { 1020 cv_broadcast(&wpipe->pipe_rcv); 1021 } 1022 1023 /* 1024 * Don't return EPIPE if I/O was successful 1025 */ 1026 if (error == EPIPE && bp->cnt == 0 && uio->uio_resid == 0) 1027 error = 0; 1028 1029 if (error == 0) 1030 getmicrotime(&wpipe->pipe_mtime); 1031 1032 /* 1033 * We have something to offer, wake up select/poll. 1034 * wpipe->pipe_map.cnt is always 0 in this point (direct write 1035 * is only done synchronously), so check only wpipe->pipe_buffer.cnt 1036 */ 1037 if (bp->cnt) 1038 pipeselwakeup(wpipe, wpipe, POLL_OUT); 1039 1040 /* 1041 * Arrange for next read(2) to do a signal. 1042 */ 1043 wpipe->pipe_state |= PIPE_SIGNALR; 1044 1045 pipeunlock(wpipe); 1046 mutex_exit(lock); 1047 return (error); 1048 } 1049 1050 /* 1051 * we implement a very minimal set of ioctls for compatibility with sockets. 1052 */ 1053 int 1054 pipe_ioctl(struct file *fp, u_long cmd, void *data) 1055 { 1056 struct pipe *pipe = fp->f_data; 1057 kmutex_t *lock = pipe->pipe_lock; 1058 1059 switch (cmd) { 1060 1061 case FIONBIO: 1062 return (0); 1063 1064 case FIOASYNC: 1065 mutex_enter(lock); 1066 if (*(int *)data) { 1067 pipe->pipe_state |= PIPE_ASYNC; 1068 } else { 1069 pipe->pipe_state &= ~PIPE_ASYNC; 1070 } 1071 mutex_exit(lock); 1072 return (0); 1073 1074 case FIONREAD: 1075 mutex_enter(lock); 1076 #ifndef PIPE_NODIRECT 1077 if (pipe->pipe_state & PIPE_DIRECTW) 1078 *(int *)data = pipe->pipe_map.cnt; 1079 else 1080 #endif 1081 *(int *)data = pipe->pipe_buffer.cnt; 1082 mutex_exit(lock); 1083 return (0); 1084 1085 case FIONWRITE: 1086 /* Look at other side */ 1087 pipe = pipe->pipe_peer; 1088 mutex_enter(lock); 1089 #ifndef PIPE_NODIRECT 1090 if (pipe->pipe_state & PIPE_DIRECTW) 1091 *(int *)data = pipe->pipe_map.cnt; 1092 else 1093 #endif 1094 *(int *)data = pipe->pipe_buffer.cnt; 1095 mutex_exit(lock); 1096 return (0); 1097 1098 case FIONSPACE: 1099 /* Look at other side */ 1100 pipe = pipe->pipe_peer; 1101 mutex_enter(lock); 1102 #ifndef PIPE_NODIRECT 1103 /* 1104 * If we're in direct-mode, we don't really have a 1105 * send queue, and any other write will block. Thus 1106 * zero seems like the best answer. 1107 */ 1108 if (pipe->pipe_state & PIPE_DIRECTW) 1109 *(int *)data = 0; 1110 else 1111 #endif 1112 *(int *)data = pipe->pipe_buffer.size - 1113 pipe->pipe_buffer.cnt; 1114 mutex_exit(lock); 1115 return (0); 1116 1117 case TIOCSPGRP: 1118 case FIOSETOWN: 1119 return fsetown(&pipe->pipe_pgid, cmd, data); 1120 1121 case TIOCGPGRP: 1122 case FIOGETOWN: 1123 return fgetown(pipe->pipe_pgid, cmd, data); 1124 1125 } 1126 return (EPASSTHROUGH); 1127 } 1128 1129 int 1130 pipe_poll(struct file *fp, int events) 1131 { 1132 struct pipe *rpipe = fp->f_data; 1133 struct pipe *wpipe; 1134 int eof = 0; 1135 int revents = 0; 1136 1137 mutex_enter(rpipe->pipe_lock); 1138 wpipe = rpipe->pipe_peer; 1139 1140 if (events & (POLLIN | POLLRDNORM)) 1141 if ((rpipe->pipe_buffer.cnt > 0) || 1142 #ifndef PIPE_NODIRECT 1143 (rpipe->pipe_state & PIPE_DIRECTR) || 1144 #endif 1145 (rpipe->pipe_state & PIPE_EOF)) 1146 revents |= events & (POLLIN | POLLRDNORM); 1147 1148 eof |= (rpipe->pipe_state & PIPE_EOF); 1149 1150 if (wpipe == NULL) 1151 revents |= events & (POLLOUT | POLLWRNORM); 1152 else { 1153 if (events & (POLLOUT | POLLWRNORM)) 1154 if ((wpipe->pipe_state & PIPE_EOF) || ( 1155 #ifndef PIPE_NODIRECT 1156 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 1157 #endif 1158 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 1159 revents |= events & (POLLOUT | POLLWRNORM); 1160 1161 eof |= (wpipe->pipe_state & PIPE_EOF); 1162 } 1163 1164 if (wpipe == NULL || eof) 1165 revents |= POLLHUP; 1166 1167 if (revents == 0) { 1168 if (events & (POLLIN | POLLRDNORM)) 1169 selrecord(curlwp, &rpipe->pipe_sel); 1170 1171 if (events & (POLLOUT | POLLWRNORM)) 1172 selrecord(curlwp, &wpipe->pipe_sel); 1173 } 1174 mutex_exit(rpipe->pipe_lock); 1175 1176 return (revents); 1177 } 1178 1179 static int 1180 pipe_stat(struct file *fp, struct stat *ub) 1181 { 1182 struct pipe *pipe = fp->f_data; 1183 1184 memset((void *)ub, 0, sizeof(*ub)); 1185 ub->st_mode = S_IFIFO | S_IRUSR | S_IWUSR; 1186 ub->st_blksize = pipe->pipe_buffer.size; 1187 if (ub->st_blksize == 0 && pipe->pipe_peer) 1188 ub->st_blksize = pipe->pipe_peer->pipe_buffer.size; 1189 ub->st_size = pipe->pipe_buffer.cnt; 1190 ub->st_blocks = (ub->st_size) ? 1 : 0; 1191 TIMEVAL_TO_TIMESPEC(&pipe->pipe_atime, &ub->st_atimespec); 1192 TIMEVAL_TO_TIMESPEC(&pipe->pipe_mtime, &ub->st_mtimespec); 1193 TIMEVAL_TO_TIMESPEC(&pipe->pipe_ctime, &ub->st_ctimespec); 1194 ub->st_uid = kauth_cred_geteuid(fp->f_cred); 1195 ub->st_gid = kauth_cred_getegid(fp->f_cred); 1196 1197 /* 1198 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 1199 * XXX (st_dev, st_ino) should be unique. 1200 */ 1201 return (0); 1202 } 1203 1204 /* ARGSUSED */ 1205 static int 1206 pipe_close(struct file *fp) 1207 { 1208 struct pipe *pipe = fp->f_data; 1209 1210 fp->f_data = NULL; 1211 pipeclose(fp, pipe); 1212 return (0); 1213 } 1214 1215 static void 1216 pipe_free_kmem(struct pipe *pipe) 1217 { 1218 1219 if (pipe->pipe_buffer.buffer != NULL) { 1220 if (pipe->pipe_buffer.size > PIPE_SIZE) 1221 atomic_dec_uint(&nbigpipe); 1222 uvm_km_free(kernel_map, 1223 (vaddr_t)pipe->pipe_buffer.buffer, 1224 pipe->pipe_buffer.size, UVM_KMF_PAGEABLE); 1225 atomic_add_int(&amountpipekva, -pipe->pipe_buffer.size); 1226 pipe->pipe_buffer.buffer = NULL; 1227 } 1228 #ifndef PIPE_NODIRECT 1229 if (pipe->pipe_map.kva != 0) { 1230 pipe_loan_free(pipe); 1231 pipe->pipe_map.cnt = 0; 1232 pipe->pipe_map.kva = 0; 1233 pipe->pipe_map.pos = 0; 1234 pipe->pipe_map.npages = 0; 1235 } 1236 #endif /* !PIPE_NODIRECT */ 1237 } 1238 1239 /* 1240 * shutdown the pipe 1241 */ 1242 static void 1243 pipeclose(struct file *fp, struct pipe *pipe) 1244 { 1245 struct pipe_mutex *mutex; 1246 kmutex_t *lock; 1247 struct pipe *ppipe; 1248 u_int refcnt; 1249 1250 if (pipe == NULL) 1251 return; 1252 1253 KASSERT(cv_is_valid(&pipe->pipe_rcv)); 1254 KASSERT(cv_is_valid(&pipe->pipe_wcv)); 1255 KASSERT(cv_is_valid(&pipe->pipe_draincv)); 1256 KASSERT(cv_is_valid(&pipe->pipe_lkcv)); 1257 1258 lock = pipe->pipe_lock; 1259 mutex_enter(lock); 1260 pipeselwakeup(pipe, pipe, POLL_HUP); 1261 1262 /* 1263 * If the other side is blocked, wake it up saying that 1264 * we want to close it down. 1265 */ 1266 pipe->pipe_state |= PIPE_EOF; 1267 if (pipe->pipe_busy) { 1268 while (pipe->pipe_busy) { 1269 cv_broadcast(&pipe->pipe_wcv); 1270 cv_wait_sig(&pipe->pipe_draincv, lock); 1271 } 1272 } 1273 1274 /* 1275 * Disconnect from peer 1276 */ 1277 if ((ppipe = pipe->pipe_peer) != NULL) { 1278 pipeselwakeup(ppipe, ppipe, POLL_HUP); 1279 ppipe->pipe_state |= PIPE_EOF; 1280 cv_broadcast(&ppipe->pipe_rcv); 1281 ppipe->pipe_peer = NULL; 1282 } 1283 1284 KASSERT((pipe->pipe_state & PIPE_LOCKFL) == 0); 1285 1286 mutex = (struct pipe_mutex *)lock; 1287 refcnt = --(mutex->pm_refcnt); 1288 KASSERT(refcnt == 0 || refcnt == 1); 1289 mutex_exit(lock); 1290 1291 /* 1292 * free resources 1293 */ 1294 pipe_free_kmem(pipe); 1295 cv_destroy(&pipe->pipe_rcv); 1296 cv_destroy(&pipe->pipe_wcv); 1297 cv_destroy(&pipe->pipe_draincv); 1298 cv_destroy(&pipe->pipe_lkcv); 1299 seldestroy(&pipe->pipe_sel); 1300 pool_cache_put(pipe_cache, pipe); 1301 if (refcnt == 0) 1302 pool_cache_put(pipe_mutex_cache, mutex); 1303 } 1304 1305 static void 1306 filt_pipedetach(struct knote *kn) 1307 { 1308 struct pipe *pipe; 1309 kmutex_t *lock; 1310 1311 pipe = ((file_t *)kn->kn_obj)->f_data; 1312 lock = pipe->pipe_lock; 1313 1314 mutex_enter(lock); 1315 1316 switch(kn->kn_filter) { 1317 case EVFILT_WRITE: 1318 /* need the peer structure, not our own */ 1319 pipe = pipe->pipe_peer; 1320 1321 /* if reader end already closed, just return */ 1322 if (pipe == NULL) { 1323 mutex_exit(lock); 1324 return; 1325 } 1326 1327 break; 1328 default: 1329 /* nothing to do */ 1330 break; 1331 } 1332 1333 #ifdef DIAGNOSTIC 1334 if (kn->kn_hook != pipe) 1335 panic("filt_pipedetach: inconsistent knote"); 1336 #endif 1337 1338 SLIST_REMOVE(&pipe->pipe_sel.sel_klist, kn, knote, kn_selnext); 1339 mutex_exit(lock); 1340 } 1341 1342 /*ARGSUSED*/ 1343 static int 1344 filt_piperead(struct knote *kn, long hint) 1345 { 1346 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_data; 1347 struct pipe *wpipe; 1348 1349 if ((hint & NOTE_SUBMIT) == 0) { 1350 mutex_enter(rpipe->pipe_lock); 1351 } 1352 wpipe = rpipe->pipe_peer; 1353 kn->kn_data = rpipe->pipe_buffer.cnt; 1354 1355 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1356 kn->kn_data = rpipe->pipe_map.cnt; 1357 1358 if ((rpipe->pipe_state & PIPE_EOF) || 1359 (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1360 kn->kn_flags |= EV_EOF; 1361 if ((hint & NOTE_SUBMIT) == 0) { 1362 mutex_exit(rpipe->pipe_lock); 1363 } 1364 return (1); 1365 } 1366 1367 if ((hint & NOTE_SUBMIT) == 0) { 1368 mutex_exit(rpipe->pipe_lock); 1369 } 1370 return (kn->kn_data > 0); 1371 } 1372 1373 /*ARGSUSED*/ 1374 static int 1375 filt_pipewrite(struct knote *kn, long hint) 1376 { 1377 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_data; 1378 struct pipe *wpipe; 1379 1380 if ((hint & NOTE_SUBMIT) == 0) { 1381 mutex_enter(rpipe->pipe_lock); 1382 } 1383 wpipe = rpipe->pipe_peer; 1384 1385 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1386 kn->kn_data = 0; 1387 kn->kn_flags |= EV_EOF; 1388 if ((hint & NOTE_SUBMIT) == 0) { 1389 mutex_exit(rpipe->pipe_lock); 1390 } 1391 return (1); 1392 } 1393 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1394 if (wpipe->pipe_state & PIPE_DIRECTW) 1395 kn->kn_data = 0; 1396 1397 if ((hint & NOTE_SUBMIT) == 0) { 1398 mutex_exit(rpipe->pipe_lock); 1399 } 1400 return (kn->kn_data >= PIPE_BUF); 1401 } 1402 1403 static const struct filterops pipe_rfiltops = 1404 { 1, NULL, filt_pipedetach, filt_piperead }; 1405 static const struct filterops pipe_wfiltops = 1406 { 1, NULL, filt_pipedetach, filt_pipewrite }; 1407 1408 /*ARGSUSED*/ 1409 static int 1410 pipe_kqfilter(struct file *fp, struct knote *kn) 1411 { 1412 struct pipe *pipe; 1413 kmutex_t *lock; 1414 1415 pipe = ((file_t *)kn->kn_obj)->f_data; 1416 lock = pipe->pipe_lock; 1417 1418 mutex_enter(lock); 1419 1420 switch (kn->kn_filter) { 1421 case EVFILT_READ: 1422 kn->kn_fop = &pipe_rfiltops; 1423 break; 1424 case EVFILT_WRITE: 1425 kn->kn_fop = &pipe_wfiltops; 1426 pipe = pipe->pipe_peer; 1427 if (pipe == NULL) { 1428 /* other end of pipe has been closed */ 1429 mutex_exit(lock); 1430 return (EBADF); 1431 } 1432 break; 1433 default: 1434 mutex_exit(lock); 1435 return (EINVAL); 1436 } 1437 1438 kn->kn_hook = pipe; 1439 SLIST_INSERT_HEAD(&pipe->pipe_sel.sel_klist, kn, kn_selnext); 1440 mutex_exit(lock); 1441 1442 return (0); 1443 } 1444 1445 /* 1446 * Handle pipe sysctls. 1447 */ 1448 SYSCTL_SETUP(sysctl_kern_pipe_setup, "sysctl kern.pipe subtree setup") 1449 { 1450 1451 sysctl_createv(clog, 0, NULL, NULL, 1452 CTLFLAG_PERMANENT, 1453 CTLTYPE_NODE, "kern", NULL, 1454 NULL, 0, NULL, 0, 1455 CTL_KERN, CTL_EOL); 1456 sysctl_createv(clog, 0, NULL, NULL, 1457 CTLFLAG_PERMANENT, 1458 CTLTYPE_NODE, "pipe", 1459 SYSCTL_DESCR("Pipe settings"), 1460 NULL, 0, NULL, 0, 1461 CTL_KERN, KERN_PIPE, CTL_EOL); 1462 1463 sysctl_createv(clog, 0, NULL, NULL, 1464 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1465 CTLTYPE_INT, "maxkvasz", 1466 SYSCTL_DESCR("Maximum amount of kernel memory to be " 1467 "used for pipes"), 1468 NULL, 0, &maxpipekva, 0, 1469 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXKVASZ, CTL_EOL); 1470 sysctl_createv(clog, 0, NULL, NULL, 1471 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1472 CTLTYPE_INT, "maxloankvasz", 1473 SYSCTL_DESCR("Limit for direct transfers via page loan"), 1474 NULL, 0, &limitpipekva, 0, 1475 CTL_KERN, KERN_PIPE, KERN_PIPE_LIMITKVA, CTL_EOL); 1476 sysctl_createv(clog, 0, NULL, NULL, 1477 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1478 CTLTYPE_INT, "maxbigpipes", 1479 SYSCTL_DESCR("Maximum number of \"big\" pipes"), 1480 NULL, 0, &maxbigpipes, 0, 1481 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXBIGPIPES, CTL_EOL); 1482 sysctl_createv(clog, 0, NULL, NULL, 1483 CTLFLAG_PERMANENT, 1484 CTLTYPE_INT, "nbigpipes", 1485 SYSCTL_DESCR("Number of \"big\" pipes"), 1486 NULL, 0, &nbigpipe, 0, 1487 CTL_KERN, KERN_PIPE, KERN_PIPE_NBIGPIPES, CTL_EOL); 1488 sysctl_createv(clog, 0, NULL, NULL, 1489 CTLFLAG_PERMANENT, 1490 CTLTYPE_INT, "kvasize", 1491 SYSCTL_DESCR("Amount of kernel memory consumed by pipe " 1492 "buffers"), 1493 NULL, 0, &amountpipekva, 0, 1494 CTL_KERN, KERN_PIPE, KERN_PIPE_KVASIZE, CTL_EOL); 1495 } 1496