1 /* $NetBSD: sys_pipe.c,v 1.148 2019/04/26 17:24:23 mlelstv Exp $ */ 2 3 /*- 4 * Copyright (c) 2003, 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Paul Kranenburg, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1996 John S. Dyson 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice immediately at the beginning of the file, without modification, 41 * this list of conditions, and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. Absolutely no warranty of function or purpose is made by the author 46 * John S. Dyson. 47 * 4. Modifications may be freely made to this file if the above conditions 48 * are met. 49 */ 50 51 /* 52 * This file contains a high-performance replacement for the socket-based 53 * pipes scheme originally used. It does not support all features of 54 * sockets, but does do everything that pipes normally do. 55 * 56 * This code has two modes of operation, a small write mode and a large 57 * write mode. The small write mode acts like conventional pipes with 58 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 59 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 60 * and PIPE_SIZE in size it is mapped read-only into the kernel address space 61 * using the UVM page loan facility from where the receiving process can copy 62 * the data directly from the pages in the sending process. 63 * 64 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 65 * happen for small transfers so that the system will not spend all of 66 * its time context switching. PIPE_SIZE is constrained by the 67 * amount of kernel virtual memory. 68 */ 69 70 #include <sys/cdefs.h> 71 __KERNEL_RCSID(0, "$NetBSD: sys_pipe.c,v 1.148 2019/04/26 17:24:23 mlelstv Exp $"); 72 73 #include <sys/param.h> 74 #include <sys/systm.h> 75 #include <sys/proc.h> 76 #include <sys/fcntl.h> 77 #include <sys/file.h> 78 #include <sys/filedesc.h> 79 #include <sys/filio.h> 80 #include <sys/kernel.h> 81 #include <sys/ttycom.h> 82 #include <sys/stat.h> 83 #include <sys/poll.h> 84 #include <sys/signalvar.h> 85 #include <sys/vnode.h> 86 #include <sys/uio.h> 87 #include <sys/select.h> 88 #include <sys/mount.h> 89 #include <sys/syscallargs.h> 90 #include <sys/sysctl.h> 91 #include <sys/kauth.h> 92 #include <sys/atomic.h> 93 #include <sys/pipe.h> 94 95 /* 96 * Use this to disable direct I/O and decrease the code size: 97 * #define PIPE_NODIRECT 98 */ 99 100 /* XXX Disabled for now; rare hangs switching between direct/buffered */ 101 #define PIPE_NODIRECT 102 103 #ifndef PIPE_NODIRECT 104 #include <uvm/uvm.h> 105 106 #if !defined(PMAP_DIRECT) 107 # define PIPE_NODIRECT /* Direct map interface not available */ 108 #endif 109 110 bool pipe_direct = true; 111 #endif 112 113 static int pipe_read(file_t *, off_t *, struct uio *, kauth_cred_t, int); 114 static int pipe_write(file_t *, off_t *, struct uio *, kauth_cred_t, int); 115 static int pipe_close(file_t *); 116 static int pipe_poll(file_t *, int); 117 static int pipe_kqfilter(file_t *, struct knote *); 118 static int pipe_stat(file_t *, struct stat *); 119 static int pipe_ioctl(file_t *, u_long, void *); 120 static void pipe_restart(file_t *); 121 122 static const struct fileops pipeops = { 123 .fo_name = "pipe", 124 .fo_read = pipe_read, 125 .fo_write = pipe_write, 126 .fo_ioctl = pipe_ioctl, 127 .fo_fcntl = fnullop_fcntl, 128 .fo_poll = pipe_poll, 129 .fo_stat = pipe_stat, 130 .fo_close = pipe_close, 131 .fo_kqfilter = pipe_kqfilter, 132 .fo_restart = pipe_restart, 133 }; 134 135 /* 136 * Default pipe buffer size(s), this can be kind-of large now because pipe 137 * space is pageable. The pipe code will try to maintain locality of 138 * reference for performance reasons, so small amounts of outstanding I/O 139 * will not wipe the cache. 140 */ 141 #define MINPIPESIZE (PIPE_SIZE / 3) 142 #define MAXPIPESIZE (2 * PIPE_SIZE / 3) 143 144 /* 145 * Limit the number of "big" pipes 146 */ 147 #define LIMITBIGPIPES 32 148 static u_int maxbigpipes = LIMITBIGPIPES; 149 static u_int nbigpipe = 0; 150 151 /* 152 * Amount of KVA consumed by pipe buffers. 153 */ 154 static u_int amountpipekva = 0; 155 156 static void pipeclose(struct pipe *); 157 static void pipe_free_kmem(struct pipe *); 158 static int pipe_create(struct pipe **, pool_cache_t); 159 static int pipelock(struct pipe *, bool); 160 static inline void pipeunlock(struct pipe *); 161 static void pipeselwakeup(struct pipe *, struct pipe *, int); 162 #ifndef PIPE_NODIRECT 163 static int pipe_direct_write(file_t *, struct pipe *, struct uio *); 164 #endif 165 static int pipespace(struct pipe *, int); 166 static int pipe_ctor(void *, void *, int); 167 static void pipe_dtor(void *, void *); 168 169 #ifndef PIPE_NODIRECT 170 static int pipe_loan_alloc(struct pipe *, int); 171 static void pipe_loan_free(struct pipe *); 172 static int pipe_direct_process_read(void *, size_t, void *); 173 #endif /* PIPE_NODIRECT */ 174 175 static pool_cache_t pipe_wr_cache; 176 static pool_cache_t pipe_rd_cache; 177 178 void 179 pipe_init(void) 180 { 181 182 /* Writer side is not automatically allocated KVA. */ 183 pipe_wr_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "pipewr", 184 NULL, IPL_NONE, pipe_ctor, pipe_dtor, NULL); 185 KASSERT(pipe_wr_cache != NULL); 186 187 /* Reader side gets preallocated KVA. */ 188 pipe_rd_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "piperd", 189 NULL, IPL_NONE, pipe_ctor, pipe_dtor, (void *)1); 190 KASSERT(pipe_rd_cache != NULL); 191 } 192 193 static int 194 pipe_ctor(void *arg, void *obj, int flags) 195 { 196 struct pipe *pipe; 197 vaddr_t va; 198 199 pipe = obj; 200 201 memset(pipe, 0, sizeof(struct pipe)); 202 if (arg != NULL) { 203 /* Preallocate space. */ 204 va = uvm_km_alloc(kernel_map, PIPE_SIZE, 0, 205 UVM_KMF_PAGEABLE | UVM_KMF_WAITVA); 206 KASSERT(va != 0); 207 pipe->pipe_kmem = va; 208 atomic_add_int(&amountpipekva, PIPE_SIZE); 209 } 210 cv_init(&pipe->pipe_rcv, "pipe_rd"); 211 cv_init(&pipe->pipe_wcv, "pipe_wr"); 212 cv_init(&pipe->pipe_draincv, "pipe_drn"); 213 cv_init(&pipe->pipe_lkcv, "pipe_lk"); 214 selinit(&pipe->pipe_sel); 215 pipe->pipe_state = PIPE_SIGNALR; 216 217 return 0; 218 } 219 220 static void 221 pipe_dtor(void *arg, void *obj) 222 { 223 struct pipe *pipe; 224 225 pipe = obj; 226 227 cv_destroy(&pipe->pipe_rcv); 228 cv_destroy(&pipe->pipe_wcv); 229 cv_destroy(&pipe->pipe_draincv); 230 cv_destroy(&pipe->pipe_lkcv); 231 seldestroy(&pipe->pipe_sel); 232 if (pipe->pipe_kmem != 0) { 233 uvm_km_free(kernel_map, pipe->pipe_kmem, PIPE_SIZE, 234 UVM_KMF_PAGEABLE); 235 atomic_add_int(&amountpipekva, -PIPE_SIZE); 236 } 237 } 238 239 /* 240 * The pipe system call for the DTYPE_PIPE type of pipes 241 */ 242 int 243 pipe1(struct lwp *l, int *fildes, int flags) 244 { 245 struct pipe *rpipe, *wpipe; 246 file_t *rf, *wf; 247 int fd, error; 248 proc_t *p; 249 250 if (flags & ~(O_CLOEXEC|O_NONBLOCK|O_NOSIGPIPE)) 251 return EINVAL; 252 p = curproc; 253 rpipe = wpipe = NULL; 254 if ((error = pipe_create(&rpipe, pipe_rd_cache)) || 255 (error = pipe_create(&wpipe, pipe_wr_cache))) { 256 goto free2; 257 } 258 rpipe->pipe_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 259 wpipe->pipe_lock = rpipe->pipe_lock; 260 mutex_obj_hold(wpipe->pipe_lock); 261 262 error = fd_allocfile(&rf, &fd); 263 if (error) 264 goto free2; 265 fildes[0] = fd; 266 267 error = fd_allocfile(&wf, &fd); 268 if (error) 269 goto free3; 270 fildes[1] = fd; 271 272 rf->f_flag = FREAD | flags; 273 rf->f_type = DTYPE_PIPE; 274 rf->f_pipe = rpipe; 275 rf->f_ops = &pipeops; 276 fd_set_exclose(l, fildes[0], (flags & O_CLOEXEC) != 0); 277 278 wf->f_flag = FWRITE | flags; 279 wf->f_type = DTYPE_PIPE; 280 wf->f_pipe = wpipe; 281 wf->f_ops = &pipeops; 282 fd_set_exclose(l, fildes[1], (flags & O_CLOEXEC) != 0); 283 284 rpipe->pipe_peer = wpipe; 285 wpipe->pipe_peer = rpipe; 286 287 fd_affix(p, rf, fildes[0]); 288 fd_affix(p, wf, fildes[1]); 289 return (0); 290 free3: 291 fd_abort(p, rf, fildes[0]); 292 free2: 293 pipeclose(wpipe); 294 pipeclose(rpipe); 295 296 return (error); 297 } 298 299 /* 300 * Allocate kva for pipe circular buffer, the space is pageable 301 * This routine will 'realloc' the size of a pipe safely, if it fails 302 * it will retain the old buffer. 303 * If it fails it will return ENOMEM. 304 */ 305 static int 306 pipespace(struct pipe *pipe, int size) 307 { 308 void *buffer; 309 310 /* 311 * Allocate pageable virtual address space. Physical memory is 312 * allocated on demand. 313 */ 314 if (size == PIPE_SIZE && pipe->pipe_kmem != 0) { 315 buffer = (void *)pipe->pipe_kmem; 316 } else { 317 buffer = (void *)uvm_km_alloc(kernel_map, round_page(size), 318 0, UVM_KMF_PAGEABLE); 319 if (buffer == NULL) 320 return (ENOMEM); 321 atomic_add_int(&amountpipekva, size); 322 } 323 324 /* free old resources if we're resizing */ 325 pipe_free_kmem(pipe); 326 pipe->pipe_buffer.buffer = buffer; 327 pipe->pipe_buffer.size = size; 328 pipe->pipe_buffer.in = 0; 329 pipe->pipe_buffer.out = 0; 330 pipe->pipe_buffer.cnt = 0; 331 return (0); 332 } 333 334 /* 335 * Initialize and allocate VM and memory for pipe. 336 */ 337 static int 338 pipe_create(struct pipe **pipep, pool_cache_t cache) 339 { 340 struct pipe *pipe; 341 int error; 342 343 pipe = pool_cache_get(cache, PR_WAITOK); 344 KASSERT(pipe != NULL); 345 *pipep = pipe; 346 error = 0; 347 getnanotime(&pipe->pipe_btime); 348 pipe->pipe_atime = pipe->pipe_mtime = pipe->pipe_btime; 349 pipe->pipe_lock = NULL; 350 if (cache == pipe_rd_cache) { 351 error = pipespace(pipe, PIPE_SIZE); 352 } else { 353 pipe->pipe_buffer.buffer = NULL; 354 pipe->pipe_buffer.size = 0; 355 pipe->pipe_buffer.in = 0; 356 pipe->pipe_buffer.out = 0; 357 pipe->pipe_buffer.cnt = 0; 358 } 359 return error; 360 } 361 362 /* 363 * Lock a pipe for I/O, blocking other access 364 * Called with pipe spin lock held. 365 */ 366 static int 367 pipelock(struct pipe *pipe, bool catch_p) 368 { 369 int error; 370 371 KASSERT(mutex_owned(pipe->pipe_lock)); 372 373 while (pipe->pipe_state & PIPE_LOCKFL) { 374 pipe->pipe_state |= PIPE_LWANT; 375 if (catch_p) { 376 error = cv_wait_sig(&pipe->pipe_lkcv, pipe->pipe_lock); 377 if (error != 0) 378 return error; 379 } else 380 cv_wait(&pipe->pipe_lkcv, pipe->pipe_lock); 381 } 382 383 pipe->pipe_state |= PIPE_LOCKFL; 384 385 return 0; 386 } 387 388 /* 389 * unlock a pipe I/O lock 390 */ 391 static inline void 392 pipeunlock(struct pipe *pipe) 393 { 394 395 KASSERT(pipe->pipe_state & PIPE_LOCKFL); 396 397 pipe->pipe_state &= ~PIPE_LOCKFL; 398 if (pipe->pipe_state & PIPE_LWANT) { 399 pipe->pipe_state &= ~PIPE_LWANT; 400 cv_broadcast(&pipe->pipe_lkcv); 401 } 402 } 403 404 /* 405 * Select/poll wakup. This also sends SIGIO to peer connected to 406 * 'sigpipe' side of pipe. 407 */ 408 static void 409 pipeselwakeup(struct pipe *selp, struct pipe *sigp, int code) 410 { 411 int band; 412 413 switch (code) { 414 case POLL_IN: 415 band = POLLIN|POLLRDNORM; 416 break; 417 case POLL_OUT: 418 band = POLLOUT|POLLWRNORM; 419 break; 420 case POLL_HUP: 421 band = POLLHUP; 422 break; 423 case POLL_ERR: 424 band = POLLERR; 425 break; 426 default: 427 band = 0; 428 #ifdef DIAGNOSTIC 429 printf("bad siginfo code %d in pipe notification.\n", code); 430 #endif 431 break; 432 } 433 434 selnotify(&selp->pipe_sel, band, NOTE_SUBMIT); 435 436 if (sigp == NULL || (sigp->pipe_state & PIPE_ASYNC) == 0) 437 return; 438 439 fownsignal(sigp->pipe_pgid, SIGIO, code, band, selp); 440 } 441 442 #ifndef PIPE_NODIRECT 443 static int 444 pipe_direct_process_read(void *va, size_t len, void *arg) 445 { 446 struct uio *uio = (struct uio *)arg; 447 448 return uiomove(va, len, uio); 449 } 450 #endif 451 452 static int 453 pipe_read(file_t *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 454 int flags) 455 { 456 struct pipe *rpipe = fp->f_pipe; 457 struct pipebuf *bp = &rpipe->pipe_buffer; 458 kmutex_t *lock = rpipe->pipe_lock; 459 int error; 460 size_t nread = 0; 461 size_t size; 462 size_t ocnt; 463 unsigned int wakeup_state = 0; 464 465 mutex_enter(lock); 466 ++rpipe->pipe_busy; 467 ocnt = bp->cnt; 468 469 again: 470 error = pipelock(rpipe, true); 471 if (error) 472 goto unlocked_error; 473 474 while (uio->uio_resid) { 475 /* 476 * Normal pipe buffer receive. 477 */ 478 if (bp->cnt > 0) { 479 size = bp->size - bp->out; 480 if (size > bp->cnt) 481 size = bp->cnt; 482 if (size > uio->uio_resid) 483 size = uio->uio_resid; 484 485 mutex_exit(lock); 486 error = uiomove((char *)bp->buffer + bp->out, size, uio); 487 mutex_enter(lock); 488 if (error) 489 break; 490 491 bp->out += size; 492 if (bp->out >= bp->size) 493 bp->out = 0; 494 495 bp->cnt -= size; 496 497 /* 498 * If there is no more to read in the pipe, reset 499 * its pointers to the beginning. This improves 500 * cache hit stats. 501 */ 502 if (bp->cnt == 0) { 503 bp->in = 0; 504 bp->out = 0; 505 } 506 nread += size; 507 continue; 508 } 509 510 #ifndef PIPE_NODIRECT 511 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) { 512 struct pipemapping * const rmap = &rpipe->pipe_map; 513 voff_t pgoff; 514 u_int pgst, npages; 515 516 /* 517 * Direct copy, bypassing a kernel buffer. 518 */ 519 KASSERT(rpipe->pipe_state & PIPE_DIRECTW); 520 521 size = MIN(rmap->cnt, uio->uio_resid); 522 523 if (size > 0) { 524 KASSERT(size > 0); 525 mutex_exit(lock); 526 527 pgst = rmap->pos >> PAGE_SHIFT; 528 pgoff = rmap->pos & PAGE_MASK; 529 npages = (size + pgoff + PAGE_SIZE - 1) >> PAGE_SHIFT; 530 KASSERTMSG(npages > 0 && (pgst + npages) <= rmap->npages, "npages %u pgst %u rmap->npages %u", npages, pgst, rmap->npages); 531 532 error = uvm_direct_process(&rmap->pgs[pgst], npages, 533 pgoff, size, pipe_direct_process_read, uio); 534 mutex_enter(lock); 535 536 nread += size; 537 rmap->pos += size; 538 rmap->cnt -= size; 539 } 540 541 if (rmap->cnt == 0) { 542 rpipe->pipe_state &= ~PIPE_DIRECTR; 543 cv_broadcast(&rpipe->pipe_wcv); 544 } 545 546 continue; 547 } 548 #endif 549 /* 550 * Break if some data was read. 551 */ 552 if (nread > 0) 553 break; 554 555 /* 556 * Detect EOF condition. 557 * Read returns 0 on EOF, no need to set error. 558 */ 559 if (rpipe->pipe_state & PIPE_EOF) 560 break; 561 562 /* 563 * Don't block on non-blocking I/O. 564 */ 565 if (fp->f_flag & FNONBLOCK) { 566 error = EAGAIN; 567 break; 568 } 569 570 /* 571 * Unlock the pipe buffer for our remaining processing. 572 * We will either break out with an error or we will 573 * sleep and relock to loop. 574 */ 575 pipeunlock(rpipe); 576 577 /* 578 * Re-check to see if more direct writes are pending. 579 */ 580 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) 581 goto again; 582 583 #if 1 /* XXX (dsl) I'm sure these aren't needed here ... */ 584 /* 585 * We want to read more, wake up select/poll. 586 */ 587 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 588 589 /* 590 * If the "write-side" is blocked, wake it up now. 591 */ 592 cv_broadcast(&rpipe->pipe_wcv); 593 #endif 594 595 if (wakeup_state & PIPE_RESTART) { 596 error = ERESTART; 597 goto unlocked_error; 598 } 599 600 /* Now wait until the pipe is filled */ 601 error = cv_wait_sig(&rpipe->pipe_rcv, lock); 602 if (error != 0) 603 goto unlocked_error; 604 wakeup_state = rpipe->pipe_state; 605 goto again; 606 } 607 608 if (error == 0) 609 getnanotime(&rpipe->pipe_atime); 610 pipeunlock(rpipe); 611 612 unlocked_error: 613 --rpipe->pipe_busy; 614 if (rpipe->pipe_busy == 0) { 615 rpipe->pipe_state &= ~PIPE_RESTART; 616 cv_broadcast(&rpipe->pipe_draincv); 617 } 618 if (bp->cnt < MINPIPESIZE) { 619 cv_broadcast(&rpipe->pipe_wcv); 620 } 621 622 /* 623 * If anything was read off the buffer, signal to the writer it's 624 * possible to write more data. Also send signal if we are here for the 625 * first time after last write. 626 */ 627 if ((bp->size - bp->cnt) >= PIPE_BUF 628 && (ocnt != bp->cnt || (rpipe->pipe_state & PIPE_SIGNALR))) { 629 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 630 rpipe->pipe_state &= ~PIPE_SIGNALR; 631 } 632 633 mutex_exit(lock); 634 return (error); 635 } 636 637 #ifndef PIPE_NODIRECT 638 /* 639 * Allocate structure for loan transfer. 640 */ 641 static int 642 pipe_loan_alloc(struct pipe *wpipe, int npages) 643 { 644 struct pipemapping * const wmap = &wpipe->pipe_map; 645 646 KASSERT(wmap->npages == 0); 647 648 if (npages > wmap->maxpages) { 649 pipe_loan_free(wpipe); 650 651 wmap->pgs = kmem_alloc(npages * sizeof(struct vm_page *), KM_NOSLEEP); 652 if (wmap->pgs == NULL) 653 return ENOMEM; 654 wmap->maxpages = npages; 655 } 656 657 wmap->npages = npages; 658 659 return (0); 660 } 661 662 /* 663 * Free resources allocated for loan transfer. 664 */ 665 static void 666 pipe_loan_free(struct pipe *wpipe) 667 { 668 struct pipemapping * const wmap = &wpipe->pipe_map; 669 670 if (wmap->maxpages > 0) { 671 kmem_free(wmap->pgs, wmap->maxpages * sizeof(struct vm_page *)); 672 wmap->pgs = NULL; 673 wmap->maxpages = 0; 674 } 675 676 wmap->npages = 0; 677 wmap->pos = 0; 678 wmap->cnt = 0; 679 } 680 681 /* 682 * NetBSD direct write, using uvm_loan() mechanism. 683 * This implements the pipe buffer write mechanism. Note that only 684 * a direct write OR a normal pipe write can be pending at any given time. 685 * If there are any characters in the pipe buffer, the direct write will 686 * be deferred until the receiving process grabs all of the bytes from 687 * the pipe buffer. Then the direct mapping write is set-up. 688 * 689 * Called with the long-term pipe lock held. 690 */ 691 static int 692 pipe_direct_write(file_t *fp, struct pipe *wpipe, struct uio *uio) 693 { 694 struct pipemapping * const wmap = &wpipe->pipe_map; 695 kmutex_t * const lock = wpipe->pipe_lock; 696 vaddr_t bbase, base, bend; 697 vsize_t blen, bcnt; 698 int error, npages; 699 voff_t bpos; 700 701 KASSERT(mutex_owned(lock)); 702 KASSERT(wmap->cnt == 0); 703 704 mutex_exit(lock); 705 706 /* 707 * Handle first PIPE_DIRECT_CHUNK bytes of buffer. Deal with buffers 708 * not aligned to PAGE_SIZE. 709 */ 710 bbase = (vaddr_t)uio->uio_iov->iov_base; 711 base = trunc_page(bbase); 712 bend = round_page(bbase + uio->uio_iov->iov_len); 713 blen = bend - base; 714 bpos = bbase - base; 715 716 if (blen > PIPE_DIRECT_CHUNK) { 717 blen = PIPE_DIRECT_CHUNK; 718 bend = base + blen; 719 bcnt = PIPE_DIRECT_CHUNK - bpos; 720 } else { 721 bcnt = uio->uio_iov->iov_len; 722 } 723 npages = atop(blen); 724 725 KASSERT((wpipe->pipe_state & (PIPE_DIRECTW | PIPE_DIRECTR)) == 0); 726 KASSERT(wmap->npages == 0); 727 728 /* Make sure page array is big enough */ 729 error = pipe_loan_alloc(wpipe, npages); 730 if (error) { 731 mutex_enter(lock); 732 return (error); 733 } 734 735 /* Loan the write buffer memory from writer process */ 736 error = uvm_loan(&uio->uio_vmspace->vm_map, base, blen, 737 wmap->pgs, UVM_LOAN_TOPAGE); 738 if (error) { 739 pipe_loan_free(wpipe); 740 mutex_enter(lock); 741 return (ENOMEM); /* so that caller fallback to ordinary write */ 742 } 743 744 /* Now we can put the pipe in direct write mode */ 745 wmap->pos = bpos; 746 wmap->cnt = bcnt; 747 748 /* 749 * But before we can let someone do a direct read, we 750 * have to wait until the pipe is drained. Release the 751 * pipe lock while we wait. 752 */ 753 mutex_enter(lock); 754 wpipe->pipe_state |= PIPE_DIRECTW; 755 pipeunlock(wpipe); 756 757 while (error == 0 && wpipe->pipe_buffer.cnt > 0) { 758 cv_broadcast(&wpipe->pipe_rcv); 759 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 760 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 761 error = EPIPE; 762 } 763 764 /* Pipe is drained; next read will off the direct buffer */ 765 wpipe->pipe_state |= PIPE_DIRECTR; 766 767 /* Wait until the reader is done */ 768 while (error == 0 && (wpipe->pipe_state & PIPE_DIRECTR)) { 769 cv_broadcast(&wpipe->pipe_rcv); 770 pipeselwakeup(wpipe, wpipe, POLL_IN); 771 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 772 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 773 error = EPIPE; 774 } 775 776 /* Take pipe out of direct write mode */ 777 wpipe->pipe_state &= ~(PIPE_DIRECTW | PIPE_DIRECTR); 778 779 /* Acquire the pipe lock and cleanup */ 780 (void)pipelock(wpipe, false); 781 782 mutex_exit(lock); 783 /* XXX what happens if the writer process exits without waiting for reader? 784 * XXX FreeBSD does a clone in this case */ 785 uvm_unloan(wmap->pgs, npages, UVM_LOAN_TOPAGE); 786 mutex_enter(lock); 787 788 if (error) { 789 pipeselwakeup(wpipe, wpipe, POLL_ERR); 790 791 /* 792 * If nothing was read from what we offered, return error 793 * straight on. Otherwise update uio resid first. Caller 794 * will deal with the error condition, returning short 795 * write, error, or restarting the write(2) as appropriate. 796 */ 797 if (wmap->cnt == bcnt) { 798 wmap->cnt = 0; 799 cv_broadcast(&wpipe->pipe_wcv); 800 return (error); 801 } 802 803 bcnt -= wmap->cnt; 804 } 805 806 uio->uio_resid -= bcnt; 807 /* uio_offset not updated, not set/used for write(2) */ 808 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + bcnt; 809 uio->uio_iov->iov_len -= bcnt; 810 if (uio->uio_iov->iov_len == 0) { 811 uio->uio_iov++; 812 uio->uio_iovcnt--; 813 } 814 815 wmap->cnt = 0; 816 return (error); 817 } 818 #endif /* !PIPE_NODIRECT */ 819 820 static int 821 pipe_write(file_t *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 822 int flags) 823 { 824 struct pipe *wpipe, *rpipe; 825 struct pipebuf *bp; 826 kmutex_t *lock; 827 int error; 828 unsigned int wakeup_state = 0; 829 830 /* We want to write to our peer */ 831 rpipe = fp->f_pipe; 832 lock = rpipe->pipe_lock; 833 error = 0; 834 835 mutex_enter(lock); 836 wpipe = rpipe->pipe_peer; 837 838 /* 839 * Detect loss of pipe read side, issue SIGPIPE if lost. 840 */ 841 if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF) != 0) { 842 mutex_exit(lock); 843 return EPIPE; 844 } 845 ++wpipe->pipe_busy; 846 847 /* Aquire the long-term pipe lock */ 848 if ((error = pipelock(wpipe, true)) != 0) { 849 --wpipe->pipe_busy; 850 if (wpipe->pipe_busy == 0) { 851 wpipe->pipe_state &= ~PIPE_RESTART; 852 cv_broadcast(&wpipe->pipe_draincv); 853 } 854 mutex_exit(lock); 855 return (error); 856 } 857 858 bp = &wpipe->pipe_buffer; 859 860 /* 861 * If it is advantageous to resize the pipe buffer, do so. 862 */ 863 if ((uio->uio_resid > PIPE_SIZE) && 864 (nbigpipe < maxbigpipes) && 865 #ifndef PIPE_NODIRECT 866 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 867 #endif 868 (bp->size <= PIPE_SIZE) && (bp->cnt == 0)) { 869 870 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) 871 atomic_inc_uint(&nbigpipe); 872 } 873 874 while (uio->uio_resid) { 875 size_t space; 876 877 #ifndef PIPE_NODIRECT 878 /* 879 * Pipe buffered writes cannot be coincidental with 880 * direct writes. Also, only one direct write can be 881 * in progress at any one time. We wait until the currently 882 * executing direct write is completed before continuing. 883 * 884 * We break out if a signal occurs or the reader goes away. 885 */ 886 while (error == 0 && wpipe->pipe_state & PIPE_DIRECTW) { 887 cv_broadcast(&wpipe->pipe_rcv); 888 pipeunlock(wpipe); 889 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 890 (void)pipelock(wpipe, false); 891 if (wpipe->pipe_state & PIPE_EOF) 892 error = EPIPE; 893 } 894 if (error) 895 break; 896 897 /* 898 * If the transfer is large, we can gain performance if 899 * we do process-to-process copies directly. 900 * If the write is non-blocking, we don't use the 901 * direct write mechanism. 902 * 903 * The direct write mechanism will detect the reader going 904 * away on us. 905 */ 906 if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) && 907 (fp->f_flag & FNONBLOCK) == 0 && 908 pipe_direct) { 909 error = pipe_direct_write(fp, wpipe, uio); 910 911 /* 912 * Break out if error occurred, unless it's ENOMEM. 913 * ENOMEM means we failed to allocate some resources 914 * for direct write, so we just fallback to ordinary 915 * write. If the direct write was successful, 916 * process rest of data via ordinary write. 917 */ 918 if (error == 0) 919 continue; 920 921 if (error != ENOMEM) 922 break; 923 } 924 #endif /* PIPE_NODIRECT */ 925 926 space = bp->size - bp->cnt; 927 928 /* Writes of size <= PIPE_BUF must be atomic. */ 929 if ((space < uio->uio_resid) && (uio->uio_resid <= PIPE_BUF)) 930 space = 0; 931 932 if (space > 0) { 933 int size; /* Transfer size */ 934 int segsize; /* first segment to transfer */ 935 936 /* 937 * Transfer size is minimum of uio transfer 938 * and free space in pipe buffer. 939 */ 940 if (space > uio->uio_resid) 941 size = uio->uio_resid; 942 else 943 size = space; 944 /* 945 * First segment to transfer is minimum of 946 * transfer size and contiguous space in 947 * pipe buffer. If first segment to transfer 948 * is less than the transfer size, we've got 949 * a wraparound in the buffer. 950 */ 951 segsize = bp->size - bp->in; 952 if (segsize > size) 953 segsize = size; 954 955 /* Transfer first segment */ 956 mutex_exit(lock); 957 error = uiomove((char *)bp->buffer + bp->in, segsize, 958 uio); 959 960 if (error == 0 && segsize < size) { 961 /* 962 * Transfer remaining part now, to 963 * support atomic writes. Wraparound 964 * happened. 965 */ 966 KASSERT(bp->in + segsize == bp->size); 967 error = uiomove(bp->buffer, 968 size - segsize, uio); 969 } 970 mutex_enter(lock); 971 if (error) 972 break; 973 974 bp->in += size; 975 if (bp->in >= bp->size) { 976 KASSERT(bp->in == size - segsize + bp->size); 977 bp->in = size - segsize; 978 } 979 980 bp->cnt += size; 981 KASSERT(bp->cnt <= bp->size); 982 wakeup_state = 0; 983 } else { 984 /* 985 * If the "read-side" has been blocked, wake it up now. 986 */ 987 cv_broadcast(&wpipe->pipe_rcv); 988 989 /* 990 * Don't block on non-blocking I/O. 991 */ 992 if (fp->f_flag & FNONBLOCK) { 993 error = EAGAIN; 994 break; 995 } 996 997 /* 998 * We have no more space and have something to offer, 999 * wake up select/poll. 1000 */ 1001 if (bp->cnt) 1002 pipeselwakeup(wpipe, wpipe, POLL_IN); 1003 1004 if (wakeup_state & PIPE_RESTART) { 1005 error = ERESTART; 1006 break; 1007 } 1008 1009 pipeunlock(wpipe); 1010 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 1011 (void)pipelock(wpipe, false); 1012 if (error != 0) 1013 break; 1014 /* 1015 * If read side wants to go away, we just issue a signal 1016 * to ourselves. 1017 */ 1018 if (wpipe->pipe_state & PIPE_EOF) { 1019 error = EPIPE; 1020 break; 1021 } 1022 wakeup_state = wpipe->pipe_state; 1023 } 1024 } 1025 1026 --wpipe->pipe_busy; 1027 if (wpipe->pipe_busy == 0) { 1028 wpipe->pipe_state &= ~PIPE_RESTART; 1029 cv_broadcast(&wpipe->pipe_draincv); 1030 } 1031 if (bp->cnt > 0) { 1032 cv_broadcast(&wpipe->pipe_rcv); 1033 } 1034 1035 /* 1036 * Don't return EPIPE if I/O was successful 1037 */ 1038 if (error == EPIPE && bp->cnt == 0 && uio->uio_resid == 0) 1039 error = 0; 1040 1041 if (error == 0) 1042 getnanotime(&wpipe->pipe_mtime); 1043 1044 /* 1045 * We have something to offer, wake up select/poll. 1046 * wmap->cnt is always 0 in this point (direct write 1047 * is only done synchronously), so check only wpipe->pipe_buffer.cnt 1048 */ 1049 if (bp->cnt) 1050 pipeselwakeup(wpipe, wpipe, POLL_IN); 1051 1052 /* 1053 * Arrange for next read(2) to do a signal. 1054 */ 1055 wpipe->pipe_state |= PIPE_SIGNALR; 1056 1057 pipeunlock(wpipe); 1058 mutex_exit(lock); 1059 return (error); 1060 } 1061 1062 /* 1063 * We implement a very minimal set of ioctls for compatibility with sockets. 1064 */ 1065 int 1066 pipe_ioctl(file_t *fp, u_long cmd, void *data) 1067 { 1068 struct pipe *pipe = fp->f_pipe; 1069 kmutex_t *lock = pipe->pipe_lock; 1070 1071 switch (cmd) { 1072 1073 case FIONBIO: 1074 return (0); 1075 1076 case FIOASYNC: 1077 mutex_enter(lock); 1078 if (*(int *)data) { 1079 pipe->pipe_state |= PIPE_ASYNC; 1080 } else { 1081 pipe->pipe_state &= ~PIPE_ASYNC; 1082 } 1083 mutex_exit(lock); 1084 return (0); 1085 1086 case FIONREAD: 1087 mutex_enter(lock); 1088 #ifndef PIPE_NODIRECT 1089 if (pipe->pipe_state & PIPE_DIRECTW) 1090 *(int *)data = pipe->pipe_map.cnt; 1091 else 1092 #endif 1093 *(int *)data = pipe->pipe_buffer.cnt; 1094 mutex_exit(lock); 1095 return (0); 1096 1097 case FIONWRITE: 1098 /* Look at other side */ 1099 mutex_enter(lock); 1100 pipe = pipe->pipe_peer; 1101 if (pipe == NULL) 1102 *(int *)data = 0; 1103 #ifndef PIPE_NODIRECT 1104 else if (pipe->pipe_state & PIPE_DIRECTW) 1105 *(int *)data = pipe->pipe_map.cnt; 1106 else 1107 #endif 1108 *(int *)data = pipe->pipe_buffer.cnt; 1109 mutex_exit(lock); 1110 return (0); 1111 1112 case FIONSPACE: 1113 /* Look at other side */ 1114 mutex_enter(lock); 1115 pipe = pipe->pipe_peer; 1116 if (pipe == NULL) 1117 *(int *)data = 0; 1118 else 1119 #ifndef PIPE_NODIRECT 1120 /* 1121 * If we're in direct-mode, we don't really have a 1122 * send queue, and any other write will block. Thus 1123 * zero seems like the best answer. 1124 */ 1125 if (pipe->pipe_state & PIPE_DIRECTW) 1126 *(int *)data = 0; 1127 else 1128 #endif 1129 *(int *)data = pipe->pipe_buffer.size - 1130 pipe->pipe_buffer.cnt; 1131 mutex_exit(lock); 1132 return (0); 1133 1134 case TIOCSPGRP: 1135 case FIOSETOWN: 1136 return fsetown(&pipe->pipe_pgid, cmd, data); 1137 1138 case TIOCGPGRP: 1139 case FIOGETOWN: 1140 return fgetown(pipe->pipe_pgid, cmd, data); 1141 1142 } 1143 return (EPASSTHROUGH); 1144 } 1145 1146 int 1147 pipe_poll(file_t *fp, int events) 1148 { 1149 struct pipe *rpipe = fp->f_pipe; 1150 struct pipe *wpipe; 1151 int eof = 0; 1152 int revents = 0; 1153 1154 mutex_enter(rpipe->pipe_lock); 1155 wpipe = rpipe->pipe_peer; 1156 1157 if (events & (POLLIN | POLLRDNORM)) 1158 if ((rpipe->pipe_buffer.cnt > 0) || 1159 #ifndef PIPE_NODIRECT 1160 (rpipe->pipe_state & PIPE_DIRECTR) || 1161 #endif 1162 (rpipe->pipe_state & PIPE_EOF)) 1163 revents |= events & (POLLIN | POLLRDNORM); 1164 1165 eof |= (rpipe->pipe_state & PIPE_EOF); 1166 1167 if (wpipe == NULL) 1168 revents |= events & (POLLOUT | POLLWRNORM); 1169 else { 1170 if (events & (POLLOUT | POLLWRNORM)) 1171 if ((wpipe->pipe_state & PIPE_EOF) || ( 1172 #ifndef PIPE_NODIRECT 1173 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 1174 #endif 1175 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 1176 revents |= events & (POLLOUT | POLLWRNORM); 1177 1178 eof |= (wpipe->pipe_state & PIPE_EOF); 1179 } 1180 1181 if (wpipe == NULL || eof) 1182 revents |= POLLHUP; 1183 1184 if (revents == 0) { 1185 if (events & (POLLIN | POLLRDNORM)) 1186 selrecord(curlwp, &rpipe->pipe_sel); 1187 1188 if (events & (POLLOUT | POLLWRNORM)) 1189 selrecord(curlwp, &wpipe->pipe_sel); 1190 } 1191 mutex_exit(rpipe->pipe_lock); 1192 1193 return (revents); 1194 } 1195 1196 static int 1197 pipe_stat(file_t *fp, struct stat *ub) 1198 { 1199 struct pipe *pipe = fp->f_pipe; 1200 1201 mutex_enter(pipe->pipe_lock); 1202 memset(ub, 0, sizeof(*ub)); 1203 ub->st_mode = S_IFIFO | S_IRUSR | S_IWUSR; 1204 ub->st_blksize = pipe->pipe_buffer.size; 1205 if (ub->st_blksize == 0 && pipe->pipe_peer) 1206 ub->st_blksize = pipe->pipe_peer->pipe_buffer.size; 1207 ub->st_size = pipe->pipe_buffer.cnt; 1208 ub->st_blocks = (ub->st_size) ? 1 : 0; 1209 ub->st_atimespec = pipe->pipe_atime; 1210 ub->st_mtimespec = pipe->pipe_mtime; 1211 ub->st_ctimespec = ub->st_birthtimespec = pipe->pipe_btime; 1212 ub->st_uid = kauth_cred_geteuid(fp->f_cred); 1213 ub->st_gid = kauth_cred_getegid(fp->f_cred); 1214 1215 /* 1216 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 1217 * XXX (st_dev, st_ino) should be unique. 1218 */ 1219 mutex_exit(pipe->pipe_lock); 1220 return 0; 1221 } 1222 1223 static int 1224 pipe_close(file_t *fp) 1225 { 1226 struct pipe *pipe = fp->f_pipe; 1227 1228 fp->f_pipe = NULL; 1229 pipeclose(pipe); 1230 return (0); 1231 } 1232 1233 static void 1234 pipe_restart(file_t *fp) 1235 { 1236 struct pipe *pipe = fp->f_pipe; 1237 1238 /* 1239 * Unblock blocked reads/writes in order to allow close() to complete. 1240 * System calls return ERESTART so that the fd is revalidated. 1241 * (Partial writes return the transfer length.) 1242 */ 1243 mutex_enter(pipe->pipe_lock); 1244 pipe->pipe_state |= PIPE_RESTART; 1245 /* Wakeup both cvs, maybe we only need one, but maybe there are some 1246 * other paths where wakeup is needed, and it saves deciding which! */ 1247 cv_broadcast(&pipe->pipe_rcv); 1248 cv_broadcast(&pipe->pipe_wcv); 1249 mutex_exit(pipe->pipe_lock); 1250 } 1251 1252 static void 1253 pipe_free_kmem(struct pipe *pipe) 1254 { 1255 1256 if (pipe->pipe_buffer.buffer != NULL) { 1257 if (pipe->pipe_buffer.size > PIPE_SIZE) { 1258 atomic_dec_uint(&nbigpipe); 1259 } 1260 if (pipe->pipe_buffer.buffer != (void *)pipe->pipe_kmem) { 1261 uvm_km_free(kernel_map, 1262 (vaddr_t)pipe->pipe_buffer.buffer, 1263 pipe->pipe_buffer.size, UVM_KMF_PAGEABLE); 1264 atomic_add_int(&amountpipekva, 1265 -pipe->pipe_buffer.size); 1266 } 1267 pipe->pipe_buffer.buffer = NULL; 1268 } 1269 #ifndef PIPE_NODIRECT 1270 if (pipe->pipe_map.npages > 0) 1271 pipe_loan_free(pipe); 1272 #endif /* !PIPE_NODIRECT */ 1273 } 1274 1275 /* 1276 * Shutdown the pipe. 1277 */ 1278 static void 1279 pipeclose(struct pipe *pipe) 1280 { 1281 kmutex_t *lock; 1282 struct pipe *ppipe; 1283 1284 if (pipe == NULL) 1285 return; 1286 1287 KASSERT(cv_is_valid(&pipe->pipe_rcv)); 1288 KASSERT(cv_is_valid(&pipe->pipe_wcv)); 1289 KASSERT(cv_is_valid(&pipe->pipe_draincv)); 1290 KASSERT(cv_is_valid(&pipe->pipe_lkcv)); 1291 1292 lock = pipe->pipe_lock; 1293 if (lock == NULL) 1294 /* Must have failed during create */ 1295 goto free_resources; 1296 1297 mutex_enter(lock); 1298 pipeselwakeup(pipe, pipe, POLL_HUP); 1299 1300 /* 1301 * If the other side is blocked, wake it up saying that 1302 * we want to close it down. 1303 */ 1304 pipe->pipe_state |= PIPE_EOF; 1305 if (pipe->pipe_busy) { 1306 while (pipe->pipe_busy) { 1307 cv_broadcast(&pipe->pipe_wcv); 1308 cv_wait_sig(&pipe->pipe_draincv, lock); 1309 } 1310 } 1311 1312 /* 1313 * Disconnect from peer. 1314 */ 1315 if ((ppipe = pipe->pipe_peer) != NULL) { 1316 pipeselwakeup(ppipe, ppipe, POLL_HUP); 1317 ppipe->pipe_state |= PIPE_EOF; 1318 cv_broadcast(&ppipe->pipe_rcv); 1319 ppipe->pipe_peer = NULL; 1320 } 1321 1322 /* 1323 * Any knote objects still left in the list are 1324 * the one attached by peer. Since no one will 1325 * traverse this list, we just clear it. 1326 */ 1327 SLIST_INIT(&pipe->pipe_sel.sel_klist); 1328 1329 KASSERT((pipe->pipe_state & PIPE_LOCKFL) == 0); 1330 mutex_exit(lock); 1331 mutex_obj_free(lock); 1332 1333 /* 1334 * Free resources. 1335 */ 1336 free_resources: 1337 pipe->pipe_pgid = 0; 1338 pipe->pipe_state = PIPE_SIGNALR; 1339 pipe->pipe_peer = NULL; 1340 pipe->pipe_lock = NULL; 1341 pipe_free_kmem(pipe); 1342 if (pipe->pipe_kmem != 0) { 1343 pool_cache_put(pipe_rd_cache, pipe); 1344 } else { 1345 pool_cache_put(pipe_wr_cache, pipe); 1346 } 1347 } 1348 1349 static void 1350 filt_pipedetach(struct knote *kn) 1351 { 1352 struct pipe *pipe; 1353 kmutex_t *lock; 1354 1355 pipe = ((file_t *)kn->kn_obj)->f_pipe; 1356 lock = pipe->pipe_lock; 1357 1358 mutex_enter(lock); 1359 1360 switch(kn->kn_filter) { 1361 case EVFILT_WRITE: 1362 /* Need the peer structure, not our own. */ 1363 pipe = pipe->pipe_peer; 1364 1365 /* If reader end already closed, just return. */ 1366 if (pipe == NULL) { 1367 mutex_exit(lock); 1368 return; 1369 } 1370 1371 break; 1372 default: 1373 /* Nothing to do. */ 1374 break; 1375 } 1376 1377 KASSERT(kn->kn_hook == pipe); 1378 SLIST_REMOVE(&pipe->pipe_sel.sel_klist, kn, knote, kn_selnext); 1379 mutex_exit(lock); 1380 } 1381 1382 static int 1383 filt_piperead(struct knote *kn, long hint) 1384 { 1385 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_pipe; 1386 struct pipe *wpipe; 1387 1388 if ((hint & NOTE_SUBMIT) == 0) { 1389 mutex_enter(rpipe->pipe_lock); 1390 } 1391 wpipe = rpipe->pipe_peer; 1392 kn->kn_data = rpipe->pipe_buffer.cnt; 1393 1394 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1395 kn->kn_data = rpipe->pipe_map.cnt; 1396 1397 if ((rpipe->pipe_state & PIPE_EOF) || 1398 (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1399 kn->kn_flags |= EV_EOF; 1400 if ((hint & NOTE_SUBMIT) == 0) { 1401 mutex_exit(rpipe->pipe_lock); 1402 } 1403 return (1); 1404 } 1405 1406 if ((hint & NOTE_SUBMIT) == 0) { 1407 mutex_exit(rpipe->pipe_lock); 1408 } 1409 return (kn->kn_data > 0); 1410 } 1411 1412 static int 1413 filt_pipewrite(struct knote *kn, long hint) 1414 { 1415 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_pipe; 1416 struct pipe *wpipe; 1417 1418 if ((hint & NOTE_SUBMIT) == 0) { 1419 mutex_enter(rpipe->pipe_lock); 1420 } 1421 wpipe = rpipe->pipe_peer; 1422 1423 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1424 kn->kn_data = 0; 1425 kn->kn_flags |= EV_EOF; 1426 if ((hint & NOTE_SUBMIT) == 0) { 1427 mutex_exit(rpipe->pipe_lock); 1428 } 1429 return (1); 1430 } 1431 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1432 if (wpipe->pipe_state & PIPE_DIRECTW) 1433 kn->kn_data = 0; 1434 1435 if ((hint & NOTE_SUBMIT) == 0) { 1436 mutex_exit(rpipe->pipe_lock); 1437 } 1438 return (kn->kn_data >= PIPE_BUF); 1439 } 1440 1441 static const struct filterops pipe_rfiltops = { 1442 .f_isfd = 1, 1443 .f_attach = NULL, 1444 .f_detach = filt_pipedetach, 1445 .f_event = filt_piperead, 1446 }; 1447 1448 static const struct filterops pipe_wfiltops = { 1449 .f_isfd = 1, 1450 .f_attach = NULL, 1451 .f_detach = filt_pipedetach, 1452 .f_event = filt_pipewrite, 1453 }; 1454 1455 static int 1456 pipe_kqfilter(file_t *fp, struct knote *kn) 1457 { 1458 struct pipe *pipe; 1459 kmutex_t *lock; 1460 1461 pipe = ((file_t *)kn->kn_obj)->f_pipe; 1462 lock = pipe->pipe_lock; 1463 1464 mutex_enter(lock); 1465 1466 switch (kn->kn_filter) { 1467 case EVFILT_READ: 1468 kn->kn_fop = &pipe_rfiltops; 1469 break; 1470 case EVFILT_WRITE: 1471 kn->kn_fop = &pipe_wfiltops; 1472 pipe = pipe->pipe_peer; 1473 if (pipe == NULL) { 1474 /* Other end of pipe has been closed. */ 1475 mutex_exit(lock); 1476 return (EBADF); 1477 } 1478 break; 1479 default: 1480 mutex_exit(lock); 1481 return (EINVAL); 1482 } 1483 1484 kn->kn_hook = pipe; 1485 SLIST_INSERT_HEAD(&pipe->pipe_sel.sel_klist, kn, kn_selnext); 1486 mutex_exit(lock); 1487 1488 return (0); 1489 } 1490 1491 /* 1492 * Handle pipe sysctls. 1493 */ 1494 SYSCTL_SETUP(sysctl_kern_pipe_setup, "sysctl kern.pipe subtree setup") 1495 { 1496 1497 sysctl_createv(clog, 0, NULL, NULL, 1498 CTLFLAG_PERMANENT, 1499 CTLTYPE_NODE, "pipe", 1500 SYSCTL_DESCR("Pipe settings"), 1501 NULL, 0, NULL, 0, 1502 CTL_KERN, KERN_PIPE, CTL_EOL); 1503 1504 sysctl_createv(clog, 0, NULL, NULL, 1505 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1506 CTLTYPE_INT, "maxbigpipes", 1507 SYSCTL_DESCR("Maximum number of \"big\" pipes"), 1508 NULL, 0, &maxbigpipes, 0, 1509 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXBIGPIPES, CTL_EOL); 1510 sysctl_createv(clog, 0, NULL, NULL, 1511 CTLFLAG_PERMANENT, 1512 CTLTYPE_INT, "nbigpipes", 1513 SYSCTL_DESCR("Number of \"big\" pipes"), 1514 NULL, 0, &nbigpipe, 0, 1515 CTL_KERN, KERN_PIPE, KERN_PIPE_NBIGPIPES, CTL_EOL); 1516 sysctl_createv(clog, 0, NULL, NULL, 1517 CTLFLAG_PERMANENT, 1518 CTLTYPE_INT, "kvasize", 1519 SYSCTL_DESCR("Amount of kernel memory consumed by pipe " 1520 "buffers"), 1521 NULL, 0, &amountpipekva, 0, 1522 CTL_KERN, KERN_PIPE, KERN_PIPE_KVASIZE, CTL_EOL); 1523 } 1524