1 /* $NetBSD: sys_pipe.c,v 1.144 2018/04/20 19:02:18 jdolecek Exp $ */ 2 3 /*- 4 * Copyright (c) 2003, 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Paul Kranenburg, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1996 John S. Dyson 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice immediately at the beginning of the file, without modification, 41 * this list of conditions, and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. Absolutely no warranty of function or purpose is made by the author 46 * John S. Dyson. 47 * 4. Modifications may be freely made to this file if the above conditions 48 * are met. 49 */ 50 51 /* 52 * This file contains a high-performance replacement for the socket-based 53 * pipes scheme originally used. It does not support all features of 54 * sockets, but does do everything that pipes normally do. 55 * 56 * This code has two modes of operation, a small write mode and a large 57 * write mode. The small write mode acts like conventional pipes with 58 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 59 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 60 * and PIPE_SIZE in size it is mapped read-only into the kernel address space 61 * using the UVM page loan facility from where the receiving process can copy 62 * the data directly from the pages in the sending process. 63 * 64 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 65 * happen for small transfers so that the system will not spend all of 66 * its time context switching. PIPE_SIZE is constrained by the 67 * amount of kernel virtual memory. 68 */ 69 70 #include <sys/cdefs.h> 71 __KERNEL_RCSID(0, "$NetBSD: sys_pipe.c,v 1.144 2018/04/20 19:02:18 jdolecek Exp $"); 72 73 #include <sys/param.h> 74 #include <sys/systm.h> 75 #include <sys/proc.h> 76 #include <sys/fcntl.h> 77 #include <sys/file.h> 78 #include <sys/filedesc.h> 79 #include <sys/filio.h> 80 #include <sys/kernel.h> 81 #include <sys/ttycom.h> 82 #include <sys/stat.h> 83 #include <sys/poll.h> 84 #include <sys/signalvar.h> 85 #include <sys/vnode.h> 86 #include <sys/uio.h> 87 #include <sys/select.h> 88 #include <sys/mount.h> 89 #include <sys/syscallargs.h> 90 #include <sys/sysctl.h> 91 #include <sys/kauth.h> 92 #include <sys/atomic.h> 93 #include <sys/pipe.h> 94 95 #include <uvm/uvm_extern.h> 96 97 /* 98 * Use this to disable direct I/O and decrease the code size: 99 * #define PIPE_NODIRECT 100 */ 101 102 /* XXX Disabled for now; rare hangs switching between direct/buffered */ 103 #define PIPE_NODIRECT 104 105 static int pipe_read(file_t *, off_t *, struct uio *, kauth_cred_t, int); 106 static int pipe_write(file_t *, off_t *, struct uio *, kauth_cred_t, int); 107 static int pipe_close(file_t *); 108 static int pipe_poll(file_t *, int); 109 static int pipe_kqfilter(file_t *, struct knote *); 110 static int pipe_stat(file_t *, struct stat *); 111 static int pipe_ioctl(file_t *, u_long, void *); 112 static void pipe_restart(file_t *); 113 114 static const struct fileops pipeops = { 115 .fo_name = "pipe", 116 .fo_read = pipe_read, 117 .fo_write = pipe_write, 118 .fo_ioctl = pipe_ioctl, 119 .fo_fcntl = fnullop_fcntl, 120 .fo_poll = pipe_poll, 121 .fo_stat = pipe_stat, 122 .fo_close = pipe_close, 123 .fo_kqfilter = pipe_kqfilter, 124 .fo_restart = pipe_restart, 125 }; 126 127 /* 128 * Default pipe buffer size(s), this can be kind-of large now because pipe 129 * space is pageable. The pipe code will try to maintain locality of 130 * reference for performance reasons, so small amounts of outstanding I/O 131 * will not wipe the cache. 132 */ 133 #define MINPIPESIZE (PIPE_SIZE / 3) 134 #define MAXPIPESIZE (2 * PIPE_SIZE / 3) 135 136 /* 137 * Maximum amount of kva for pipes -- this is kind-of a soft limit, but 138 * is there so that on large systems, we don't exhaust it. 139 */ 140 #define MAXPIPEKVA (8 * 1024 * 1024) 141 static u_int maxpipekva = MAXPIPEKVA; 142 143 /* 144 * Limit for direct transfers, we cannot, of course limit 145 * the amount of kva for pipes in general though. 146 */ 147 #define LIMITPIPEKVA (16 * 1024 * 1024) 148 static u_int limitpipekva = LIMITPIPEKVA; 149 150 /* 151 * Limit the number of "big" pipes 152 */ 153 #define LIMITBIGPIPES 32 154 static u_int maxbigpipes = LIMITBIGPIPES; 155 static u_int nbigpipe = 0; 156 157 /* 158 * Amount of KVA consumed by pipe buffers. 159 */ 160 static u_int amountpipekva = 0; 161 162 static void pipeclose(struct pipe *); 163 static void pipe_free_kmem(struct pipe *); 164 static int pipe_create(struct pipe **, pool_cache_t); 165 static int pipelock(struct pipe *, bool); 166 static inline void pipeunlock(struct pipe *); 167 static void pipeselwakeup(struct pipe *, struct pipe *, int); 168 #ifndef PIPE_NODIRECT 169 static int pipe_direct_write(file_t *, struct pipe *, struct uio *); 170 #endif 171 static int pipespace(struct pipe *, int); 172 static int pipe_ctor(void *, void *, int); 173 static void pipe_dtor(void *, void *); 174 175 #ifndef PIPE_NODIRECT 176 static int pipe_loan_alloc(struct pipe *, int); 177 static void pipe_loan_free(struct pipe *); 178 #endif /* PIPE_NODIRECT */ 179 180 static pool_cache_t pipe_wr_cache; 181 static pool_cache_t pipe_rd_cache; 182 183 void 184 pipe_init(void) 185 { 186 187 /* Writer side is not automatically allocated KVA. */ 188 pipe_wr_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "pipewr", 189 NULL, IPL_NONE, pipe_ctor, pipe_dtor, NULL); 190 KASSERT(pipe_wr_cache != NULL); 191 192 /* Reader side gets preallocated KVA. */ 193 pipe_rd_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "piperd", 194 NULL, IPL_NONE, pipe_ctor, pipe_dtor, (void *)1); 195 KASSERT(pipe_rd_cache != NULL); 196 } 197 198 static int 199 pipe_ctor(void *arg, void *obj, int flags) 200 { 201 struct pipe *pipe; 202 vaddr_t va; 203 204 pipe = obj; 205 206 memset(pipe, 0, sizeof(struct pipe)); 207 if (arg != NULL) { 208 /* Preallocate space. */ 209 va = uvm_km_alloc(kernel_map, PIPE_SIZE, 0, 210 UVM_KMF_PAGEABLE | UVM_KMF_WAITVA); 211 KASSERT(va != 0); 212 pipe->pipe_kmem = va; 213 atomic_add_int(&amountpipekva, PIPE_SIZE); 214 } 215 cv_init(&pipe->pipe_rcv, "pipe_rd"); 216 cv_init(&pipe->pipe_wcv, "pipe_wr"); 217 cv_init(&pipe->pipe_draincv, "pipe_drn"); 218 cv_init(&pipe->pipe_lkcv, "pipe_lk"); 219 selinit(&pipe->pipe_sel); 220 pipe->pipe_state = PIPE_SIGNALR; 221 222 return 0; 223 } 224 225 static void 226 pipe_dtor(void *arg, void *obj) 227 { 228 struct pipe *pipe; 229 230 pipe = obj; 231 232 cv_destroy(&pipe->pipe_rcv); 233 cv_destroy(&pipe->pipe_wcv); 234 cv_destroy(&pipe->pipe_draincv); 235 cv_destroy(&pipe->pipe_lkcv); 236 seldestroy(&pipe->pipe_sel); 237 if (pipe->pipe_kmem != 0) { 238 uvm_km_free(kernel_map, pipe->pipe_kmem, PIPE_SIZE, 239 UVM_KMF_PAGEABLE); 240 atomic_add_int(&amountpipekva, -PIPE_SIZE); 241 } 242 } 243 244 /* 245 * The pipe system call for the DTYPE_PIPE type of pipes 246 */ 247 int 248 pipe1(struct lwp *l, int *fildes, int flags) 249 { 250 struct pipe *rpipe, *wpipe; 251 file_t *rf, *wf; 252 int fd, error; 253 proc_t *p; 254 255 if (flags & ~(O_CLOEXEC|O_NONBLOCK|O_NOSIGPIPE)) 256 return EINVAL; 257 p = curproc; 258 rpipe = wpipe = NULL; 259 if ((error = pipe_create(&rpipe, pipe_rd_cache)) || 260 (error = pipe_create(&wpipe, pipe_wr_cache))) { 261 goto free2; 262 } 263 rpipe->pipe_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 264 wpipe->pipe_lock = rpipe->pipe_lock; 265 mutex_obj_hold(wpipe->pipe_lock); 266 267 error = fd_allocfile(&rf, &fd); 268 if (error) 269 goto free2; 270 fildes[0] = fd; 271 272 error = fd_allocfile(&wf, &fd); 273 if (error) 274 goto free3; 275 fildes[1] = fd; 276 277 rf->f_flag = FREAD | flags; 278 rf->f_type = DTYPE_PIPE; 279 rf->f_pipe = rpipe; 280 rf->f_ops = &pipeops; 281 fd_set_exclose(l, fildes[0], (flags & O_CLOEXEC) != 0); 282 283 wf->f_flag = FWRITE | flags; 284 wf->f_type = DTYPE_PIPE; 285 wf->f_pipe = wpipe; 286 wf->f_ops = &pipeops; 287 fd_set_exclose(l, fildes[1], (flags & O_CLOEXEC) != 0); 288 289 rpipe->pipe_peer = wpipe; 290 wpipe->pipe_peer = rpipe; 291 292 fd_affix(p, rf, fildes[0]); 293 fd_affix(p, wf, fildes[1]); 294 return (0); 295 free3: 296 fd_abort(p, rf, fildes[0]); 297 free2: 298 pipeclose(wpipe); 299 pipeclose(rpipe); 300 301 return (error); 302 } 303 304 /* 305 * Allocate kva for pipe circular buffer, the space is pageable 306 * This routine will 'realloc' the size of a pipe safely, if it fails 307 * it will retain the old buffer. 308 * If it fails it will return ENOMEM. 309 */ 310 static int 311 pipespace(struct pipe *pipe, int size) 312 { 313 void *buffer; 314 315 /* 316 * Allocate pageable virtual address space. Physical memory is 317 * allocated on demand. 318 */ 319 if (size == PIPE_SIZE && pipe->pipe_kmem != 0) { 320 buffer = (void *)pipe->pipe_kmem; 321 } else { 322 buffer = (void *)uvm_km_alloc(kernel_map, round_page(size), 323 0, UVM_KMF_PAGEABLE); 324 if (buffer == NULL) 325 return (ENOMEM); 326 atomic_add_int(&amountpipekva, size); 327 } 328 329 /* free old resources if we're resizing */ 330 pipe_free_kmem(pipe); 331 pipe->pipe_buffer.buffer = buffer; 332 pipe->pipe_buffer.size = size; 333 pipe->pipe_buffer.in = 0; 334 pipe->pipe_buffer.out = 0; 335 pipe->pipe_buffer.cnt = 0; 336 return (0); 337 } 338 339 /* 340 * Initialize and allocate VM and memory for pipe. 341 */ 342 static int 343 pipe_create(struct pipe **pipep, pool_cache_t cache) 344 { 345 struct pipe *pipe; 346 int error; 347 348 pipe = pool_cache_get(cache, PR_WAITOK); 349 KASSERT(pipe != NULL); 350 *pipep = pipe; 351 error = 0; 352 getnanotime(&pipe->pipe_btime); 353 pipe->pipe_atime = pipe->pipe_mtime = pipe->pipe_btime; 354 pipe->pipe_lock = NULL; 355 if (cache == pipe_rd_cache) { 356 error = pipespace(pipe, PIPE_SIZE); 357 } else { 358 pipe->pipe_buffer.buffer = NULL; 359 pipe->pipe_buffer.size = 0; 360 pipe->pipe_buffer.in = 0; 361 pipe->pipe_buffer.out = 0; 362 pipe->pipe_buffer.cnt = 0; 363 } 364 return error; 365 } 366 367 /* 368 * Lock a pipe for I/O, blocking other access 369 * Called with pipe spin lock held. 370 */ 371 static int 372 pipelock(struct pipe *pipe, bool catch_p) 373 { 374 int error; 375 376 KASSERT(mutex_owned(pipe->pipe_lock)); 377 378 while (pipe->pipe_state & PIPE_LOCKFL) { 379 pipe->pipe_state |= PIPE_LWANT; 380 if (catch_p) { 381 error = cv_wait_sig(&pipe->pipe_lkcv, pipe->pipe_lock); 382 if (error != 0) 383 return error; 384 } else 385 cv_wait(&pipe->pipe_lkcv, pipe->pipe_lock); 386 } 387 388 pipe->pipe_state |= PIPE_LOCKFL; 389 390 return 0; 391 } 392 393 /* 394 * unlock a pipe I/O lock 395 */ 396 static inline void 397 pipeunlock(struct pipe *pipe) 398 { 399 400 KASSERT(pipe->pipe_state & PIPE_LOCKFL); 401 402 pipe->pipe_state &= ~PIPE_LOCKFL; 403 if (pipe->pipe_state & PIPE_LWANT) { 404 pipe->pipe_state &= ~PIPE_LWANT; 405 cv_broadcast(&pipe->pipe_lkcv); 406 } 407 } 408 409 /* 410 * Select/poll wakup. This also sends SIGIO to peer connected to 411 * 'sigpipe' side of pipe. 412 */ 413 static void 414 pipeselwakeup(struct pipe *selp, struct pipe *sigp, int code) 415 { 416 int band; 417 418 switch (code) { 419 case POLL_IN: 420 band = POLLIN|POLLRDNORM; 421 break; 422 case POLL_OUT: 423 band = POLLOUT|POLLWRNORM; 424 break; 425 case POLL_HUP: 426 band = POLLHUP; 427 break; 428 case POLL_ERR: 429 band = POLLERR; 430 break; 431 default: 432 band = 0; 433 #ifdef DIAGNOSTIC 434 printf("bad siginfo code %d in pipe notification.\n", code); 435 #endif 436 break; 437 } 438 439 selnotify(&selp->pipe_sel, band, NOTE_SUBMIT); 440 441 if (sigp == NULL || (sigp->pipe_state & PIPE_ASYNC) == 0) 442 return; 443 444 fownsignal(sigp->pipe_pgid, SIGIO, code, band, selp); 445 } 446 447 static int 448 pipe_read(file_t *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 449 int flags) 450 { 451 struct pipe *rpipe = fp->f_pipe; 452 struct pipebuf *bp = &rpipe->pipe_buffer; 453 kmutex_t *lock = rpipe->pipe_lock; 454 int error; 455 size_t nread = 0; 456 size_t size; 457 size_t ocnt; 458 unsigned int wakeup_state = 0; 459 460 mutex_enter(lock); 461 ++rpipe->pipe_busy; 462 ocnt = bp->cnt; 463 464 again: 465 error = pipelock(rpipe, true); 466 if (error) 467 goto unlocked_error; 468 469 while (uio->uio_resid) { 470 /* 471 * Normal pipe buffer receive. 472 */ 473 if (bp->cnt > 0) { 474 size = bp->size - bp->out; 475 if (size > bp->cnt) 476 size = bp->cnt; 477 if (size > uio->uio_resid) 478 size = uio->uio_resid; 479 480 mutex_exit(lock); 481 error = uiomove((char *)bp->buffer + bp->out, size, uio); 482 mutex_enter(lock); 483 if (error) 484 break; 485 486 bp->out += size; 487 if (bp->out >= bp->size) 488 bp->out = 0; 489 490 bp->cnt -= size; 491 492 /* 493 * If there is no more to read in the pipe, reset 494 * its pointers to the beginning. This improves 495 * cache hit stats. 496 */ 497 if (bp->cnt == 0) { 498 bp->in = 0; 499 bp->out = 0; 500 } 501 nread += size; 502 continue; 503 } 504 505 #ifndef PIPE_NODIRECT 506 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) { 507 struct pipemapping * const rmap = &rpipe->pipe_map; 508 /* 509 * Direct copy, bypassing a kernel buffer. 510 */ 511 void *va; 512 u_int gen; 513 514 KASSERT(rpipe->pipe_state & PIPE_DIRECTW); 515 516 size = rmap->cnt; 517 if (size > uio->uio_resid) 518 size = uio->uio_resid; 519 520 va = (char *)rmap->kva + rmap->pos; 521 gen = rmap->egen; 522 mutex_exit(lock); 523 524 /* 525 * Consume emap and read the data from loaned pages. 526 */ 527 uvm_emap_consume(gen); 528 error = uiomove(va, size, uio); 529 530 mutex_enter(lock); 531 if (error) 532 break; 533 nread += size; 534 rmap->pos += size; 535 rmap->cnt -= size; 536 if (rmap->cnt == 0) { 537 rpipe->pipe_state &= ~PIPE_DIRECTR; 538 cv_broadcast(&rpipe->pipe_wcv); 539 } 540 continue; 541 } 542 #endif 543 /* 544 * Break if some data was read. 545 */ 546 if (nread > 0) 547 break; 548 549 /* 550 * Detect EOF condition. 551 * Read returns 0 on EOF, no need to set error. 552 */ 553 if (rpipe->pipe_state & PIPE_EOF) 554 break; 555 556 /* 557 * Don't block on non-blocking I/O. 558 */ 559 if (fp->f_flag & FNONBLOCK) { 560 error = EAGAIN; 561 break; 562 } 563 564 /* 565 * Unlock the pipe buffer for our remaining processing. 566 * We will either break out with an error or we will 567 * sleep and relock to loop. 568 */ 569 pipeunlock(rpipe); 570 571 /* 572 * Re-check to see if more direct writes are pending. 573 */ 574 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) 575 goto again; 576 577 #if 1 /* XXX (dsl) I'm sure these aren't needed here ... */ 578 /* 579 * We want to read more, wake up select/poll. 580 */ 581 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 582 583 /* 584 * If the "write-side" is blocked, wake it up now. 585 */ 586 cv_broadcast(&rpipe->pipe_wcv); 587 #endif 588 589 if (wakeup_state & PIPE_RESTART) { 590 error = ERESTART; 591 goto unlocked_error; 592 } 593 594 /* Now wait until the pipe is filled */ 595 error = cv_wait_sig(&rpipe->pipe_rcv, lock); 596 if (error != 0) 597 goto unlocked_error; 598 wakeup_state = rpipe->pipe_state; 599 goto again; 600 } 601 602 if (error == 0) 603 getnanotime(&rpipe->pipe_atime); 604 pipeunlock(rpipe); 605 606 unlocked_error: 607 --rpipe->pipe_busy; 608 if (rpipe->pipe_busy == 0) { 609 rpipe->pipe_state &= ~PIPE_RESTART; 610 cv_broadcast(&rpipe->pipe_draincv); 611 } 612 if (bp->cnt < MINPIPESIZE) { 613 cv_broadcast(&rpipe->pipe_wcv); 614 } 615 616 /* 617 * If anything was read off the buffer, signal to the writer it's 618 * possible to write more data. Also send signal if we are here for the 619 * first time after last write. 620 */ 621 if ((bp->size - bp->cnt) >= PIPE_BUF 622 && (ocnt != bp->cnt || (rpipe->pipe_state & PIPE_SIGNALR))) { 623 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 624 rpipe->pipe_state &= ~PIPE_SIGNALR; 625 } 626 627 mutex_exit(lock); 628 return (error); 629 } 630 631 #ifndef PIPE_NODIRECT 632 /* 633 * Allocate structure for loan transfer. 634 */ 635 static int 636 pipe_loan_alloc(struct pipe *wpipe, int npages) 637 { 638 struct pipemapping * const wmap = &wpipe->pipe_map; 639 const vsize_t len = ptoa(npages); 640 641 atomic_add_int(&amountpipekva, len); 642 wmap->kva = uvm_km_alloc(kernel_map, len, 0, 643 UVM_KMF_COLORMATCH | UVM_KMF_VAONLY | UVM_KMF_WAITVA); 644 if (wmap->kva == 0) { 645 atomic_add_int(&amountpipekva, -len); 646 return (ENOMEM); 647 } 648 649 wmap->npages = npages; 650 wmap->pgs = kmem_alloc(npages * sizeof(struct vm_page *), KM_SLEEP); 651 return (0); 652 } 653 654 /* 655 * Free resources allocated for loan transfer. 656 */ 657 static void 658 pipe_loan_free(struct pipe *wpipe) 659 { 660 struct pipemapping * const wmap = &wpipe->pipe_map; 661 const vsize_t len = ptoa(wmap->npages); 662 663 uvm_emap_remove(wmap->kva, len); /* XXX */ 664 uvm_km_free(kernel_map, wmap->kva, len, UVM_KMF_VAONLY); 665 wmap->kva = 0; 666 atomic_add_int(&amountpipekva, -len); 667 kmem_free(wmap->pgs, wmap->npages * sizeof(struct vm_page *)); 668 wmap->pgs = NULL; 669 #if 0 670 wmap->npages = 0; 671 wmap->pos = 0; 672 wmap->cnt = 0; 673 #endif 674 } 675 676 /* 677 * NetBSD direct write, using uvm_loan() mechanism. 678 * This implements the pipe buffer write mechanism. Note that only 679 * a direct write OR a normal pipe write can be pending at any given time. 680 * If there are any characters in the pipe buffer, the direct write will 681 * be deferred until the receiving process grabs all of the bytes from 682 * the pipe buffer. Then the direct mapping write is set-up. 683 * 684 * Called with the long-term pipe lock held. 685 */ 686 static int 687 pipe_direct_write(file_t *fp, struct pipe *wpipe, struct uio *uio) 688 { 689 struct pipemapping * const wmap = &wpipe->pipe_map; 690 kmutex_t * const lock = wpipe->pipe_lock; 691 struct vm_page **pgs; 692 vaddr_t bbase, base, bend; 693 vsize_t blen, bcnt; 694 int error, npages; 695 voff_t bpos; 696 u_int starting_color; 697 698 KASSERT(mutex_owned(wpipe->pipe_lock)); 699 KASSERT(wmap->cnt == 0); 700 701 mutex_exit(lock); 702 703 /* 704 * Handle first PIPE_CHUNK_SIZE bytes of buffer. Deal with buffers 705 * not aligned to PAGE_SIZE. 706 */ 707 bbase = (vaddr_t)uio->uio_iov->iov_base; 708 base = trunc_page(bbase); 709 bend = round_page(bbase + uio->uio_iov->iov_len); 710 blen = bend - base; 711 bpos = bbase - base; 712 713 if (blen > PIPE_DIRECT_CHUNK) { 714 blen = PIPE_DIRECT_CHUNK; 715 bend = base + blen; 716 bcnt = PIPE_DIRECT_CHUNK - bpos; 717 } else { 718 bcnt = uio->uio_iov->iov_len; 719 } 720 npages = atop(blen); 721 starting_color = atop(base) & uvmexp.colormask; 722 723 /* 724 * Free the old kva if we need more pages than we have 725 * allocated. 726 */ 727 if (wmap->kva != 0 && starting_color + npages > wmap->npages) 728 pipe_loan_free(wpipe); 729 730 /* Allocate new kva. */ 731 if (wmap->kva == 0) { 732 error = pipe_loan_alloc(wpipe, starting_color + npages); 733 if (error) { 734 mutex_enter(lock); 735 return (error); 736 } 737 } 738 739 /* Loan the write buffer memory from writer process */ 740 pgs = wmap->pgs + starting_color; 741 error = uvm_loan(&uio->uio_vmspace->vm_map, base, blen, 742 pgs, UVM_LOAN_TOPAGE); 743 if (error) { 744 pipe_loan_free(wpipe); 745 mutex_enter(lock); 746 return (ENOMEM); /* so that caller fallback to ordinary write */ 747 } 748 749 /* Enter the loaned pages to KVA, produce new emap generation number. */ 750 uvm_emap_enter(wmap->kva + ptoa(starting_color), pgs, npages, 751 VM_PROT_READ); 752 wmap->egen = uvm_emap_produce(); 753 754 /* Now we can put the pipe in direct write mode */ 755 wmap->pos = bpos + ptoa(starting_color); 756 wmap->cnt = bcnt; 757 758 /* 759 * But before we can let someone do a direct read, we 760 * have to wait until the pipe is drained. Release the 761 * pipe lock while we wait. 762 */ 763 mutex_enter(lock); 764 wpipe->pipe_state |= PIPE_DIRECTW; 765 pipeunlock(wpipe); 766 767 while (error == 0 && wpipe->pipe_buffer.cnt > 0) { 768 cv_broadcast(&wpipe->pipe_rcv); 769 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 770 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 771 error = EPIPE; 772 } 773 774 /* Pipe is drained; next read will off the direct buffer */ 775 wpipe->pipe_state |= PIPE_DIRECTR; 776 777 /* Wait until the reader is done */ 778 while (error == 0 && (wpipe->pipe_state & PIPE_DIRECTR)) { 779 cv_broadcast(&wpipe->pipe_rcv); 780 pipeselwakeup(wpipe, wpipe, POLL_IN); 781 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 782 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 783 error = EPIPE; 784 } 785 786 /* Take pipe out of direct write mode */ 787 wpipe->pipe_state &= ~(PIPE_DIRECTW | PIPE_DIRECTR); 788 789 /* Acquire the pipe lock and cleanup */ 790 (void)pipelock(wpipe, false); 791 mutex_exit(lock); 792 793 if (pgs != NULL) { 794 /* XXX: uvm_emap_remove */ 795 uvm_unloan(pgs, npages, UVM_LOAN_TOPAGE); 796 } 797 if (error || amountpipekva > maxpipekva) 798 pipe_loan_free(wpipe); 799 800 mutex_enter(lock); 801 if (error) { 802 pipeselwakeup(wpipe, wpipe, POLL_ERR); 803 804 /* 805 * If nothing was read from what we offered, return error 806 * straight on. Otherwise update uio resid first. Caller 807 * will deal with the error condition, returning short 808 * write, error, or restarting the write(2) as appropriate. 809 */ 810 if (wmap->cnt == bcnt) { 811 wmap->cnt = 0; 812 cv_broadcast(&wpipe->pipe_wcv); 813 return (error); 814 } 815 816 bcnt -= wpipe->cnt; 817 } 818 819 uio->uio_resid -= bcnt; 820 /* uio_offset not updated, not set/used for write(2) */ 821 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + bcnt; 822 uio->uio_iov->iov_len -= bcnt; 823 if (uio->uio_iov->iov_len == 0) { 824 uio->uio_iov++; 825 uio->uio_iovcnt--; 826 } 827 828 wmap->cnt = 0; 829 return (error); 830 } 831 #endif /* !PIPE_NODIRECT */ 832 833 static int 834 pipe_write(file_t *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 835 int flags) 836 { 837 struct pipe *wpipe, *rpipe; 838 struct pipebuf *bp; 839 kmutex_t *lock; 840 int error; 841 unsigned int wakeup_state = 0; 842 843 /* We want to write to our peer */ 844 rpipe = fp->f_pipe; 845 lock = rpipe->pipe_lock; 846 error = 0; 847 848 mutex_enter(lock); 849 wpipe = rpipe->pipe_peer; 850 851 /* 852 * Detect loss of pipe read side, issue SIGPIPE if lost. 853 */ 854 if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF) != 0) { 855 mutex_exit(lock); 856 return EPIPE; 857 } 858 ++wpipe->pipe_busy; 859 860 /* Aquire the long-term pipe lock */ 861 if ((error = pipelock(wpipe, true)) != 0) { 862 --wpipe->pipe_busy; 863 if (wpipe->pipe_busy == 0) { 864 wpipe->pipe_state &= ~PIPE_RESTART; 865 cv_broadcast(&wpipe->pipe_draincv); 866 } 867 mutex_exit(lock); 868 return (error); 869 } 870 871 bp = &wpipe->pipe_buffer; 872 873 /* 874 * If it is advantageous to resize the pipe buffer, do so. 875 */ 876 if ((uio->uio_resid > PIPE_SIZE) && 877 (nbigpipe < maxbigpipes) && 878 #ifndef PIPE_NODIRECT 879 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 880 #endif 881 (bp->size <= PIPE_SIZE) && (bp->cnt == 0)) { 882 883 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) 884 atomic_inc_uint(&nbigpipe); 885 } 886 887 while (uio->uio_resid) { 888 size_t space; 889 890 #ifndef PIPE_NODIRECT 891 /* 892 * Pipe buffered writes cannot be coincidental with 893 * direct writes. Also, only one direct write can be 894 * in progress at any one time. We wait until the currently 895 * executing direct write is completed before continuing. 896 * 897 * We break out if a signal occurs or the reader goes away. 898 */ 899 while (error == 0 && wpipe->pipe_state & PIPE_DIRECTW) { 900 cv_broadcast(&wpipe->pipe_rcv); 901 pipeunlock(wpipe); 902 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 903 (void)pipelock(wpipe, false); 904 if (wpipe->pipe_state & PIPE_EOF) 905 error = EPIPE; 906 } 907 if (error) 908 break; 909 910 /* 911 * If the transfer is large, we can gain performance if 912 * we do process-to-process copies directly. 913 * If the write is non-blocking, we don't use the 914 * direct write mechanism. 915 * 916 * The direct write mechanism will detect the reader going 917 * away on us. 918 */ 919 if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) && 920 (fp->f_flag & FNONBLOCK) == 0 && 921 (wmap->kva || (amountpipekva < limitpipekva))) { 922 error = pipe_direct_write(fp, wpipe, uio); 923 924 /* 925 * Break out if error occurred, unless it's ENOMEM. 926 * ENOMEM means we failed to allocate some resources 927 * for direct write, so we just fallback to ordinary 928 * write. If the direct write was successful, 929 * process rest of data via ordinary write. 930 */ 931 if (error == 0) 932 continue; 933 934 if (error != ENOMEM) 935 break; 936 } 937 #endif /* PIPE_NODIRECT */ 938 939 space = bp->size - bp->cnt; 940 941 /* Writes of size <= PIPE_BUF must be atomic. */ 942 if ((space < uio->uio_resid) && (uio->uio_resid <= PIPE_BUF)) 943 space = 0; 944 945 if (space > 0) { 946 int size; /* Transfer size */ 947 int segsize; /* first segment to transfer */ 948 949 /* 950 * Transfer size is minimum of uio transfer 951 * and free space in pipe buffer. 952 */ 953 if (space > uio->uio_resid) 954 size = uio->uio_resid; 955 else 956 size = space; 957 /* 958 * First segment to transfer is minimum of 959 * transfer size and contiguous space in 960 * pipe buffer. If first segment to transfer 961 * is less than the transfer size, we've got 962 * a wraparound in the buffer. 963 */ 964 segsize = bp->size - bp->in; 965 if (segsize > size) 966 segsize = size; 967 968 /* Transfer first segment */ 969 mutex_exit(lock); 970 error = uiomove((char *)bp->buffer + bp->in, segsize, 971 uio); 972 973 if (error == 0 && segsize < size) { 974 /* 975 * Transfer remaining part now, to 976 * support atomic writes. Wraparound 977 * happened. 978 */ 979 KASSERT(bp->in + segsize == bp->size); 980 error = uiomove(bp->buffer, 981 size - segsize, uio); 982 } 983 mutex_enter(lock); 984 if (error) 985 break; 986 987 bp->in += size; 988 if (bp->in >= bp->size) { 989 KASSERT(bp->in == size - segsize + bp->size); 990 bp->in = size - segsize; 991 } 992 993 bp->cnt += size; 994 KASSERT(bp->cnt <= bp->size); 995 wakeup_state = 0; 996 } else { 997 /* 998 * If the "read-side" has been blocked, wake it up now. 999 */ 1000 cv_broadcast(&wpipe->pipe_rcv); 1001 1002 /* 1003 * Don't block on non-blocking I/O. 1004 */ 1005 if (fp->f_flag & FNONBLOCK) { 1006 error = EAGAIN; 1007 break; 1008 } 1009 1010 /* 1011 * We have no more space and have something to offer, 1012 * wake up select/poll. 1013 */ 1014 if (bp->cnt) 1015 pipeselwakeup(wpipe, wpipe, POLL_IN); 1016 1017 if (wakeup_state & PIPE_RESTART) { 1018 error = ERESTART; 1019 break; 1020 } 1021 1022 pipeunlock(wpipe); 1023 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 1024 (void)pipelock(wpipe, false); 1025 if (error != 0) 1026 break; 1027 /* 1028 * If read side wants to go away, we just issue a signal 1029 * to ourselves. 1030 */ 1031 if (wpipe->pipe_state & PIPE_EOF) { 1032 error = EPIPE; 1033 break; 1034 } 1035 wakeup_state = wpipe->pipe_state; 1036 } 1037 } 1038 1039 --wpipe->pipe_busy; 1040 if (wpipe->pipe_busy == 0) { 1041 wpipe->pipe_state &= ~PIPE_RESTART; 1042 cv_broadcast(&wpipe->pipe_draincv); 1043 } 1044 if (bp->cnt > 0) { 1045 cv_broadcast(&wpipe->pipe_rcv); 1046 } 1047 1048 /* 1049 * Don't return EPIPE if I/O was successful 1050 */ 1051 if (error == EPIPE && bp->cnt == 0 && uio->uio_resid == 0) 1052 error = 0; 1053 1054 if (error == 0) 1055 getnanotime(&wpipe->pipe_mtime); 1056 1057 /* 1058 * We have something to offer, wake up select/poll. 1059 * wmap->cnt is always 0 in this point (direct write 1060 * is only done synchronously), so check only wpipe->pipe_buffer.cnt 1061 */ 1062 if (bp->cnt) 1063 pipeselwakeup(wpipe, wpipe, POLL_IN); 1064 1065 /* 1066 * Arrange for next read(2) to do a signal. 1067 */ 1068 wpipe->pipe_state |= PIPE_SIGNALR; 1069 1070 pipeunlock(wpipe); 1071 mutex_exit(lock); 1072 return (error); 1073 } 1074 1075 /* 1076 * We implement a very minimal set of ioctls for compatibility with sockets. 1077 */ 1078 int 1079 pipe_ioctl(file_t *fp, u_long cmd, void *data) 1080 { 1081 struct pipe *pipe = fp->f_pipe; 1082 kmutex_t *lock = pipe->pipe_lock; 1083 1084 switch (cmd) { 1085 1086 case FIONBIO: 1087 return (0); 1088 1089 case FIOASYNC: 1090 mutex_enter(lock); 1091 if (*(int *)data) { 1092 pipe->pipe_state |= PIPE_ASYNC; 1093 } else { 1094 pipe->pipe_state &= ~PIPE_ASYNC; 1095 } 1096 mutex_exit(lock); 1097 return (0); 1098 1099 case FIONREAD: 1100 mutex_enter(lock); 1101 #ifndef PIPE_NODIRECT 1102 if (pipe->pipe_state & PIPE_DIRECTW) 1103 *(int *)data = pipe->pipe_map.cnt; 1104 else 1105 #endif 1106 *(int *)data = pipe->pipe_buffer.cnt; 1107 mutex_exit(lock); 1108 return (0); 1109 1110 case FIONWRITE: 1111 /* Look at other side */ 1112 pipe = pipe->pipe_peer; 1113 mutex_enter(lock); 1114 #ifndef PIPE_NODIRECT 1115 if (pipe->pipe_state & PIPE_DIRECTW) 1116 *(int *)data = pipe->pipe_map.cnt; 1117 else 1118 #endif 1119 *(int *)data = pipe->pipe_buffer.cnt; 1120 mutex_exit(lock); 1121 return (0); 1122 1123 case FIONSPACE: 1124 /* Look at other side */ 1125 pipe = pipe->pipe_peer; 1126 mutex_enter(lock); 1127 #ifndef PIPE_NODIRECT 1128 /* 1129 * If we're in direct-mode, we don't really have a 1130 * send queue, and any other write will block. Thus 1131 * zero seems like the best answer. 1132 */ 1133 if (pipe->pipe_state & PIPE_DIRECTW) 1134 *(int *)data = 0; 1135 else 1136 #endif 1137 *(int *)data = pipe->pipe_buffer.size - 1138 pipe->pipe_buffer.cnt; 1139 mutex_exit(lock); 1140 return (0); 1141 1142 case TIOCSPGRP: 1143 case FIOSETOWN: 1144 return fsetown(&pipe->pipe_pgid, cmd, data); 1145 1146 case TIOCGPGRP: 1147 case FIOGETOWN: 1148 return fgetown(pipe->pipe_pgid, cmd, data); 1149 1150 } 1151 return (EPASSTHROUGH); 1152 } 1153 1154 int 1155 pipe_poll(file_t *fp, int events) 1156 { 1157 struct pipe *rpipe = fp->f_pipe; 1158 struct pipe *wpipe; 1159 int eof = 0; 1160 int revents = 0; 1161 1162 mutex_enter(rpipe->pipe_lock); 1163 wpipe = rpipe->pipe_peer; 1164 1165 if (events & (POLLIN | POLLRDNORM)) 1166 if ((rpipe->pipe_buffer.cnt > 0) || 1167 #ifndef PIPE_NODIRECT 1168 (rpipe->pipe_state & PIPE_DIRECTR) || 1169 #endif 1170 (rpipe->pipe_state & PIPE_EOF)) 1171 revents |= events & (POLLIN | POLLRDNORM); 1172 1173 eof |= (rpipe->pipe_state & PIPE_EOF); 1174 1175 if (wpipe == NULL) 1176 revents |= events & (POLLOUT | POLLWRNORM); 1177 else { 1178 if (events & (POLLOUT | POLLWRNORM)) 1179 if ((wpipe->pipe_state & PIPE_EOF) || ( 1180 #ifndef PIPE_NODIRECT 1181 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 1182 #endif 1183 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 1184 revents |= events & (POLLOUT | POLLWRNORM); 1185 1186 eof |= (wpipe->pipe_state & PIPE_EOF); 1187 } 1188 1189 if (wpipe == NULL || eof) 1190 revents |= POLLHUP; 1191 1192 if (revents == 0) { 1193 if (events & (POLLIN | POLLRDNORM)) 1194 selrecord(curlwp, &rpipe->pipe_sel); 1195 1196 if (events & (POLLOUT | POLLWRNORM)) 1197 selrecord(curlwp, &wpipe->pipe_sel); 1198 } 1199 mutex_exit(rpipe->pipe_lock); 1200 1201 return (revents); 1202 } 1203 1204 static int 1205 pipe_stat(file_t *fp, struct stat *ub) 1206 { 1207 struct pipe *pipe = fp->f_pipe; 1208 1209 mutex_enter(pipe->pipe_lock); 1210 memset(ub, 0, sizeof(*ub)); 1211 ub->st_mode = S_IFIFO | S_IRUSR | S_IWUSR; 1212 ub->st_blksize = pipe->pipe_buffer.size; 1213 if (ub->st_blksize == 0 && pipe->pipe_peer) 1214 ub->st_blksize = pipe->pipe_peer->pipe_buffer.size; 1215 ub->st_size = pipe->pipe_buffer.cnt; 1216 ub->st_blocks = (ub->st_size) ? 1 : 0; 1217 ub->st_atimespec = pipe->pipe_atime; 1218 ub->st_mtimespec = pipe->pipe_mtime; 1219 ub->st_ctimespec = ub->st_birthtimespec = pipe->pipe_btime; 1220 ub->st_uid = kauth_cred_geteuid(fp->f_cred); 1221 ub->st_gid = kauth_cred_getegid(fp->f_cred); 1222 1223 /* 1224 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 1225 * XXX (st_dev, st_ino) should be unique. 1226 */ 1227 mutex_exit(pipe->pipe_lock); 1228 return 0; 1229 } 1230 1231 static int 1232 pipe_close(file_t *fp) 1233 { 1234 struct pipe *pipe = fp->f_pipe; 1235 1236 fp->f_pipe = NULL; 1237 pipeclose(pipe); 1238 return (0); 1239 } 1240 1241 static void 1242 pipe_restart(file_t *fp) 1243 { 1244 struct pipe *pipe = fp->f_pipe; 1245 1246 /* 1247 * Unblock blocked reads/writes in order to allow close() to complete. 1248 * System calls return ERESTART so that the fd is revalidated. 1249 * (Partial writes return the transfer length.) 1250 */ 1251 mutex_enter(pipe->pipe_lock); 1252 pipe->pipe_state |= PIPE_RESTART; 1253 /* Wakeup both cvs, maybe we only need one, but maybe there are some 1254 * other paths where wakeup is needed, and it saves deciding which! */ 1255 cv_broadcast(&pipe->pipe_rcv); 1256 cv_broadcast(&pipe->pipe_wcv); 1257 mutex_exit(pipe->pipe_lock); 1258 } 1259 1260 static void 1261 pipe_free_kmem(struct pipe *pipe) 1262 { 1263 1264 if (pipe->pipe_buffer.buffer != NULL) { 1265 if (pipe->pipe_buffer.size > PIPE_SIZE) { 1266 atomic_dec_uint(&nbigpipe); 1267 } 1268 if (pipe->pipe_buffer.buffer != (void *)pipe->pipe_kmem) { 1269 uvm_km_free(kernel_map, 1270 (vaddr_t)pipe->pipe_buffer.buffer, 1271 pipe->pipe_buffer.size, UVM_KMF_PAGEABLE); 1272 atomic_add_int(&amountpipekva, 1273 -pipe->pipe_buffer.size); 1274 } 1275 pipe->pipe_buffer.buffer = NULL; 1276 } 1277 #ifndef PIPE_NODIRECT 1278 if (pipe->pipe_map.kva != 0) { 1279 pipe_loan_free(pipe); 1280 pipe->pipe_map.cnt = 0; 1281 pipe->pipe_map.pos = 0; 1282 pipe->pipe_map.npages = 0; 1283 } 1284 #endif /* !PIPE_NODIRECT */ 1285 } 1286 1287 /* 1288 * Shutdown the pipe. 1289 */ 1290 static void 1291 pipeclose(struct pipe *pipe) 1292 { 1293 kmutex_t *lock; 1294 struct pipe *ppipe; 1295 1296 if (pipe == NULL) 1297 return; 1298 1299 KASSERT(cv_is_valid(&pipe->pipe_rcv)); 1300 KASSERT(cv_is_valid(&pipe->pipe_wcv)); 1301 KASSERT(cv_is_valid(&pipe->pipe_draincv)); 1302 KASSERT(cv_is_valid(&pipe->pipe_lkcv)); 1303 1304 lock = pipe->pipe_lock; 1305 if (lock == NULL) 1306 /* Must have failed during create */ 1307 goto free_resources; 1308 1309 mutex_enter(lock); 1310 pipeselwakeup(pipe, pipe, POLL_HUP); 1311 1312 /* 1313 * If the other side is blocked, wake it up saying that 1314 * we want to close it down. 1315 */ 1316 pipe->pipe_state |= PIPE_EOF; 1317 if (pipe->pipe_busy) { 1318 while (pipe->pipe_busy) { 1319 cv_broadcast(&pipe->pipe_wcv); 1320 cv_wait_sig(&pipe->pipe_draincv, lock); 1321 } 1322 } 1323 1324 /* 1325 * Disconnect from peer. 1326 */ 1327 if ((ppipe = pipe->pipe_peer) != NULL) { 1328 pipeselwakeup(ppipe, ppipe, POLL_HUP); 1329 ppipe->pipe_state |= PIPE_EOF; 1330 cv_broadcast(&ppipe->pipe_rcv); 1331 ppipe->pipe_peer = NULL; 1332 } 1333 1334 /* 1335 * Any knote objects still left in the list are 1336 * the one attached by peer. Since no one will 1337 * traverse this list, we just clear it. 1338 */ 1339 SLIST_INIT(&pipe->pipe_sel.sel_klist); 1340 1341 KASSERT((pipe->pipe_state & PIPE_LOCKFL) == 0); 1342 mutex_exit(lock); 1343 mutex_obj_free(lock); 1344 1345 /* 1346 * Free resources. 1347 */ 1348 free_resources: 1349 pipe->pipe_pgid = 0; 1350 pipe->pipe_state = PIPE_SIGNALR; 1351 pipe_free_kmem(pipe); 1352 if (pipe->pipe_kmem != 0) { 1353 pool_cache_put(pipe_rd_cache, pipe); 1354 } else { 1355 pool_cache_put(pipe_wr_cache, pipe); 1356 } 1357 } 1358 1359 static void 1360 filt_pipedetach(struct knote *kn) 1361 { 1362 struct pipe *pipe; 1363 kmutex_t *lock; 1364 1365 pipe = ((file_t *)kn->kn_obj)->f_pipe; 1366 lock = pipe->pipe_lock; 1367 1368 mutex_enter(lock); 1369 1370 switch(kn->kn_filter) { 1371 case EVFILT_WRITE: 1372 /* Need the peer structure, not our own. */ 1373 pipe = pipe->pipe_peer; 1374 1375 /* If reader end already closed, just return. */ 1376 if (pipe == NULL) { 1377 mutex_exit(lock); 1378 return; 1379 } 1380 1381 break; 1382 default: 1383 /* Nothing to do. */ 1384 break; 1385 } 1386 1387 KASSERT(kn->kn_hook == pipe); 1388 SLIST_REMOVE(&pipe->pipe_sel.sel_klist, kn, knote, kn_selnext); 1389 mutex_exit(lock); 1390 } 1391 1392 static int 1393 filt_piperead(struct knote *kn, long hint) 1394 { 1395 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_pipe; 1396 struct pipe *wpipe; 1397 1398 if ((hint & NOTE_SUBMIT) == 0) { 1399 mutex_enter(rpipe->pipe_lock); 1400 } 1401 wpipe = rpipe->pipe_peer; 1402 kn->kn_data = rpipe->pipe_buffer.cnt; 1403 1404 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1405 kn->kn_data = rpipe->pipe_map.cnt; 1406 1407 if ((rpipe->pipe_state & PIPE_EOF) || 1408 (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1409 kn->kn_flags |= EV_EOF; 1410 if ((hint & NOTE_SUBMIT) == 0) { 1411 mutex_exit(rpipe->pipe_lock); 1412 } 1413 return (1); 1414 } 1415 1416 if ((hint & NOTE_SUBMIT) == 0) { 1417 mutex_exit(rpipe->pipe_lock); 1418 } 1419 return (kn->kn_data > 0); 1420 } 1421 1422 static int 1423 filt_pipewrite(struct knote *kn, long hint) 1424 { 1425 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_pipe; 1426 struct pipe *wpipe; 1427 1428 if ((hint & NOTE_SUBMIT) == 0) { 1429 mutex_enter(rpipe->pipe_lock); 1430 } 1431 wpipe = rpipe->pipe_peer; 1432 1433 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1434 kn->kn_data = 0; 1435 kn->kn_flags |= EV_EOF; 1436 if ((hint & NOTE_SUBMIT) == 0) { 1437 mutex_exit(rpipe->pipe_lock); 1438 } 1439 return (1); 1440 } 1441 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1442 if (wpipe->pipe_state & PIPE_DIRECTW) 1443 kn->kn_data = 0; 1444 1445 if ((hint & NOTE_SUBMIT) == 0) { 1446 mutex_exit(rpipe->pipe_lock); 1447 } 1448 return (kn->kn_data >= PIPE_BUF); 1449 } 1450 1451 static const struct filterops pipe_rfiltops = { 1452 .f_isfd = 1, 1453 .f_attach = NULL, 1454 .f_detach = filt_pipedetach, 1455 .f_event = filt_piperead, 1456 }; 1457 1458 static const struct filterops pipe_wfiltops = { 1459 .f_isfd = 1, 1460 .f_attach = NULL, 1461 .f_detach = filt_pipedetach, 1462 .f_event = filt_pipewrite, 1463 }; 1464 1465 static int 1466 pipe_kqfilter(file_t *fp, struct knote *kn) 1467 { 1468 struct pipe *pipe; 1469 kmutex_t *lock; 1470 1471 pipe = ((file_t *)kn->kn_obj)->f_pipe; 1472 lock = pipe->pipe_lock; 1473 1474 mutex_enter(lock); 1475 1476 switch (kn->kn_filter) { 1477 case EVFILT_READ: 1478 kn->kn_fop = &pipe_rfiltops; 1479 break; 1480 case EVFILT_WRITE: 1481 kn->kn_fop = &pipe_wfiltops; 1482 pipe = pipe->pipe_peer; 1483 if (pipe == NULL) { 1484 /* Other end of pipe has been closed. */ 1485 mutex_exit(lock); 1486 return (EBADF); 1487 } 1488 break; 1489 default: 1490 mutex_exit(lock); 1491 return (EINVAL); 1492 } 1493 1494 kn->kn_hook = pipe; 1495 SLIST_INSERT_HEAD(&pipe->pipe_sel.sel_klist, kn, kn_selnext); 1496 mutex_exit(lock); 1497 1498 return (0); 1499 } 1500 1501 /* 1502 * Handle pipe sysctls. 1503 */ 1504 SYSCTL_SETUP(sysctl_kern_pipe_setup, "sysctl kern.pipe subtree setup") 1505 { 1506 1507 sysctl_createv(clog, 0, NULL, NULL, 1508 CTLFLAG_PERMANENT, 1509 CTLTYPE_NODE, "pipe", 1510 SYSCTL_DESCR("Pipe settings"), 1511 NULL, 0, NULL, 0, 1512 CTL_KERN, KERN_PIPE, CTL_EOL); 1513 1514 sysctl_createv(clog, 0, NULL, NULL, 1515 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1516 CTLTYPE_INT, "maxkvasz", 1517 SYSCTL_DESCR("Maximum amount of kernel memory to be " 1518 "used for pipes"), 1519 NULL, 0, &maxpipekva, 0, 1520 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXKVASZ, CTL_EOL); 1521 sysctl_createv(clog, 0, NULL, NULL, 1522 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1523 CTLTYPE_INT, "maxloankvasz", 1524 SYSCTL_DESCR("Limit for direct transfers via page loan"), 1525 NULL, 0, &limitpipekva, 0, 1526 CTL_KERN, KERN_PIPE, KERN_PIPE_LIMITKVA, CTL_EOL); 1527 sysctl_createv(clog, 0, NULL, NULL, 1528 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1529 CTLTYPE_INT, "maxbigpipes", 1530 SYSCTL_DESCR("Maximum number of \"big\" pipes"), 1531 NULL, 0, &maxbigpipes, 0, 1532 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXBIGPIPES, CTL_EOL); 1533 sysctl_createv(clog, 0, NULL, NULL, 1534 CTLFLAG_PERMANENT, 1535 CTLTYPE_INT, "nbigpipes", 1536 SYSCTL_DESCR("Number of \"big\" pipes"), 1537 NULL, 0, &nbigpipe, 0, 1538 CTL_KERN, KERN_PIPE, KERN_PIPE_NBIGPIPES, CTL_EOL); 1539 sysctl_createv(clog, 0, NULL, NULL, 1540 CTLFLAG_PERMANENT, 1541 CTLTYPE_INT, "kvasize", 1542 SYSCTL_DESCR("Amount of kernel memory consumed by pipe " 1543 "buffers"), 1544 NULL, 0, &amountpipekva, 0, 1545 CTL_KERN, KERN_PIPE, KERN_PIPE_KVASIZE, CTL_EOL); 1546 } 1547