1 /* $NetBSD: sys_pipe.c,v 1.132 2011/07/15 14:50:19 christos Exp $ */ 2 3 /*- 4 * Copyright (c) 2003, 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Paul Kranenburg, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1996 John S. Dyson 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice immediately at the beginning of the file, without modification, 41 * this list of conditions, and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. Absolutely no warranty of function or purpose is made by the author 46 * John S. Dyson. 47 * 4. Modifications may be freely made to this file if the above conditions 48 * are met. 49 */ 50 51 /* 52 * This file contains a high-performance replacement for the socket-based 53 * pipes scheme originally used. It does not support all features of 54 * sockets, but does do everything that pipes normally do. 55 * 56 * This code has two modes of operation, a small write mode and a large 57 * write mode. The small write mode acts like conventional pipes with 58 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 59 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 60 * and PIPE_SIZE in size it is mapped read-only into the kernel address space 61 * using the UVM page loan facility from where the receiving process can copy 62 * the data directly from the pages in the sending process. 63 * 64 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 65 * happen for small transfers so that the system will not spend all of 66 * its time context switching. PIPE_SIZE is constrained by the 67 * amount of kernel virtual memory. 68 */ 69 70 #include <sys/cdefs.h> 71 __KERNEL_RCSID(0, "$NetBSD: sys_pipe.c,v 1.132 2011/07/15 14:50:19 christos Exp $"); 72 73 #include <sys/param.h> 74 #include <sys/systm.h> 75 #include <sys/proc.h> 76 #include <sys/fcntl.h> 77 #include <sys/file.h> 78 #include <sys/filedesc.h> 79 #include <sys/filio.h> 80 #include <sys/kernel.h> 81 #include <sys/ttycom.h> 82 #include <sys/stat.h> 83 #include <sys/poll.h> 84 #include <sys/signalvar.h> 85 #include <sys/vnode.h> 86 #include <sys/uio.h> 87 #include <sys/select.h> 88 #include <sys/mount.h> 89 #include <sys/syscallargs.h> 90 #include <sys/sysctl.h> 91 #include <sys/kauth.h> 92 #include <sys/atomic.h> 93 #include <sys/pipe.h> 94 95 #include <uvm/uvm_extern.h> 96 97 /* 98 * Use this to disable direct I/O and decrease the code size: 99 * #define PIPE_NODIRECT 100 */ 101 102 /* XXX Disabled for now; rare hangs switching between direct/buffered */ 103 #define PIPE_NODIRECT 104 105 static int pipe_read(file_t *, off_t *, struct uio *, kauth_cred_t, int); 106 static int pipe_write(file_t *, off_t *, struct uio *, kauth_cred_t, int); 107 static int pipe_close(file_t *); 108 static int pipe_poll(file_t *, int); 109 static int pipe_kqfilter(file_t *, struct knote *); 110 static int pipe_stat(file_t *, struct stat *); 111 static int pipe_ioctl(file_t *, u_long, void *); 112 static void pipe_restart(file_t *); 113 114 static const struct fileops pipeops = { 115 .fo_read = pipe_read, 116 .fo_write = pipe_write, 117 .fo_ioctl = pipe_ioctl, 118 .fo_fcntl = fnullop_fcntl, 119 .fo_poll = pipe_poll, 120 .fo_stat = pipe_stat, 121 .fo_close = pipe_close, 122 .fo_kqfilter = pipe_kqfilter, 123 .fo_restart = pipe_restart, 124 }; 125 126 /* 127 * Default pipe buffer size(s), this can be kind-of large now because pipe 128 * space is pageable. The pipe code will try to maintain locality of 129 * reference for performance reasons, so small amounts of outstanding I/O 130 * will not wipe the cache. 131 */ 132 #define MINPIPESIZE (PIPE_SIZE / 3) 133 #define MAXPIPESIZE (2 * PIPE_SIZE / 3) 134 135 /* 136 * Maximum amount of kva for pipes -- this is kind-of a soft limit, but 137 * is there so that on large systems, we don't exhaust it. 138 */ 139 #define MAXPIPEKVA (8 * 1024 * 1024) 140 static u_int maxpipekva = MAXPIPEKVA; 141 142 /* 143 * Limit for direct transfers, we cannot, of course limit 144 * the amount of kva for pipes in general though. 145 */ 146 #define LIMITPIPEKVA (16 * 1024 * 1024) 147 static u_int limitpipekva = LIMITPIPEKVA; 148 149 /* 150 * Limit the number of "big" pipes 151 */ 152 #define LIMITBIGPIPES 32 153 static u_int maxbigpipes = LIMITBIGPIPES; 154 static u_int nbigpipe = 0; 155 156 /* 157 * Amount of KVA consumed by pipe buffers. 158 */ 159 static u_int amountpipekva = 0; 160 161 static void pipeclose(struct pipe *); 162 static void pipe_free_kmem(struct pipe *); 163 static int pipe_create(struct pipe **, pool_cache_t); 164 static int pipelock(struct pipe *, int); 165 static inline void pipeunlock(struct pipe *); 166 static void pipeselwakeup(struct pipe *, struct pipe *, int); 167 #ifndef PIPE_NODIRECT 168 static int pipe_direct_write(file_t *, struct pipe *, struct uio *); 169 #endif 170 static int pipespace(struct pipe *, int); 171 static int pipe_ctor(void *, void *, int); 172 static void pipe_dtor(void *, void *); 173 174 #ifndef PIPE_NODIRECT 175 static int pipe_loan_alloc(struct pipe *, int); 176 static void pipe_loan_free(struct pipe *); 177 #endif /* PIPE_NODIRECT */ 178 179 static pool_cache_t pipe_wr_cache; 180 static pool_cache_t pipe_rd_cache; 181 182 void 183 pipe_init(void) 184 { 185 186 /* Writer side is not automatically allocated KVA. */ 187 pipe_wr_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "pipewr", 188 NULL, IPL_NONE, pipe_ctor, pipe_dtor, NULL); 189 KASSERT(pipe_wr_cache != NULL); 190 191 /* Reader side gets preallocated KVA. */ 192 pipe_rd_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "piperd", 193 NULL, IPL_NONE, pipe_ctor, pipe_dtor, (void *)1); 194 KASSERT(pipe_rd_cache != NULL); 195 } 196 197 static int 198 pipe_ctor(void *arg, void *obj, int flags) 199 { 200 struct pipe *pipe; 201 vaddr_t va; 202 203 pipe = obj; 204 205 memset(pipe, 0, sizeof(struct pipe)); 206 if (arg != NULL) { 207 /* Preallocate space. */ 208 va = uvm_km_alloc(kernel_map, PIPE_SIZE, 0, 209 UVM_KMF_PAGEABLE | UVM_KMF_WAITVA); 210 KASSERT(va != 0); 211 pipe->pipe_kmem = va; 212 atomic_add_int(&amountpipekva, PIPE_SIZE); 213 } 214 cv_init(&pipe->pipe_rcv, "pipe_rd"); 215 cv_init(&pipe->pipe_wcv, "pipe_wr"); 216 cv_init(&pipe->pipe_draincv, "pipe_drn"); 217 cv_init(&pipe->pipe_lkcv, "pipe_lk"); 218 selinit(&pipe->pipe_sel); 219 pipe->pipe_state = PIPE_SIGNALR; 220 221 return 0; 222 } 223 224 static void 225 pipe_dtor(void *arg, void *obj) 226 { 227 struct pipe *pipe; 228 229 pipe = obj; 230 231 cv_destroy(&pipe->pipe_rcv); 232 cv_destroy(&pipe->pipe_wcv); 233 cv_destroy(&pipe->pipe_draincv); 234 cv_destroy(&pipe->pipe_lkcv); 235 seldestroy(&pipe->pipe_sel); 236 if (pipe->pipe_kmem != 0) { 237 uvm_km_free(kernel_map, pipe->pipe_kmem, PIPE_SIZE, 238 UVM_KMF_PAGEABLE); 239 atomic_add_int(&amountpipekva, -PIPE_SIZE); 240 } 241 } 242 243 /* 244 * The pipe system call for the DTYPE_PIPE type of pipes 245 */ 246 int 247 pipe1(struct lwp *l, register_t *retval, int flags) 248 { 249 struct pipe *rpipe, *wpipe; 250 file_t *rf, *wf; 251 int fd, error; 252 proc_t *p; 253 254 if (flags & ~(O_CLOEXEC|O_NONBLOCK)) 255 return EINVAL; 256 p = curproc; 257 rpipe = wpipe = NULL; 258 if (pipe_create(&rpipe, pipe_rd_cache) || 259 pipe_create(&wpipe, pipe_wr_cache)) { 260 error = ENOMEM; 261 goto free2; 262 } 263 rpipe->pipe_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 264 wpipe->pipe_lock = rpipe->pipe_lock; 265 mutex_obj_hold(wpipe->pipe_lock); 266 267 error = fd_allocfile(&rf, &fd); 268 if (error) 269 goto free2; 270 retval[0] = fd; 271 rf->f_flag = FREAD | flags; 272 rf->f_type = DTYPE_PIPE; 273 rf->f_data = (void *)rpipe; 274 rf->f_ops = &pipeops; 275 276 error = fd_allocfile(&wf, &fd); 277 if (error) 278 goto free3; 279 retval[1] = fd; 280 wf->f_flag = FWRITE | flags; 281 wf->f_type = DTYPE_PIPE; 282 wf->f_data = (void *)wpipe; 283 wf->f_ops = &pipeops; 284 285 rpipe->pipe_peer = wpipe; 286 wpipe->pipe_peer = rpipe; 287 288 fd_affix(p, rf, (int)retval[0]); 289 fd_affix(p, wf, (int)retval[1]); 290 return (0); 291 free3: 292 fd_abort(p, rf, (int)retval[0]); 293 free2: 294 pipeclose(wpipe); 295 pipeclose(rpipe); 296 297 return (error); 298 } 299 300 /* 301 * Allocate kva for pipe circular buffer, the space is pageable 302 * This routine will 'realloc' the size of a pipe safely, if it fails 303 * it will retain the old buffer. 304 * If it fails it will return ENOMEM. 305 */ 306 static int 307 pipespace(struct pipe *pipe, int size) 308 { 309 void *buffer; 310 311 /* 312 * Allocate pageable virtual address space. Physical memory is 313 * allocated on demand. 314 */ 315 if (size == PIPE_SIZE && pipe->pipe_kmem != 0) { 316 buffer = (void *)pipe->pipe_kmem; 317 } else { 318 buffer = (void *)uvm_km_alloc(kernel_map, round_page(size), 319 0, UVM_KMF_PAGEABLE); 320 if (buffer == NULL) 321 return (ENOMEM); 322 atomic_add_int(&amountpipekva, size); 323 } 324 325 /* free old resources if we're resizing */ 326 pipe_free_kmem(pipe); 327 pipe->pipe_buffer.buffer = buffer; 328 pipe->pipe_buffer.size = size; 329 pipe->pipe_buffer.in = 0; 330 pipe->pipe_buffer.out = 0; 331 pipe->pipe_buffer.cnt = 0; 332 return (0); 333 } 334 335 /* 336 * Initialize and allocate VM and memory for pipe. 337 */ 338 static int 339 pipe_create(struct pipe **pipep, pool_cache_t cache) 340 { 341 struct pipe *pipe; 342 int error; 343 344 pipe = pool_cache_get(cache, PR_WAITOK); 345 KASSERT(pipe != NULL); 346 *pipep = pipe; 347 error = 0; 348 getnanotime(&pipe->pipe_btime); 349 pipe->pipe_atime = pipe->pipe_mtime = pipe->pipe_btime; 350 pipe->pipe_lock = NULL; 351 if (cache == pipe_rd_cache) { 352 error = pipespace(pipe, PIPE_SIZE); 353 } else { 354 pipe->pipe_buffer.buffer = NULL; 355 pipe->pipe_buffer.size = 0; 356 pipe->pipe_buffer.in = 0; 357 pipe->pipe_buffer.out = 0; 358 pipe->pipe_buffer.cnt = 0; 359 } 360 return error; 361 } 362 363 /* 364 * Lock a pipe for I/O, blocking other access 365 * Called with pipe spin lock held. 366 */ 367 static int 368 pipelock(struct pipe *pipe, int catch) 369 { 370 int error; 371 372 KASSERT(mutex_owned(pipe->pipe_lock)); 373 374 while (pipe->pipe_state & PIPE_LOCKFL) { 375 pipe->pipe_state |= PIPE_LWANT; 376 if (catch) { 377 error = cv_wait_sig(&pipe->pipe_lkcv, pipe->pipe_lock); 378 if (error != 0) 379 return error; 380 } else 381 cv_wait(&pipe->pipe_lkcv, pipe->pipe_lock); 382 } 383 384 pipe->pipe_state |= PIPE_LOCKFL; 385 386 return 0; 387 } 388 389 /* 390 * unlock a pipe I/O lock 391 */ 392 static inline void 393 pipeunlock(struct pipe *pipe) 394 { 395 396 KASSERT(pipe->pipe_state & PIPE_LOCKFL); 397 398 pipe->pipe_state &= ~PIPE_LOCKFL; 399 if (pipe->pipe_state & PIPE_LWANT) { 400 pipe->pipe_state &= ~PIPE_LWANT; 401 cv_broadcast(&pipe->pipe_lkcv); 402 } 403 } 404 405 /* 406 * Select/poll wakup. This also sends SIGIO to peer connected to 407 * 'sigpipe' side of pipe. 408 */ 409 static void 410 pipeselwakeup(struct pipe *selp, struct pipe *sigp, int code) 411 { 412 int band; 413 414 switch (code) { 415 case POLL_IN: 416 band = POLLIN|POLLRDNORM; 417 break; 418 case POLL_OUT: 419 band = POLLOUT|POLLWRNORM; 420 break; 421 case POLL_HUP: 422 band = POLLHUP; 423 break; 424 case POLL_ERR: 425 band = POLLERR; 426 break; 427 default: 428 band = 0; 429 #ifdef DIAGNOSTIC 430 printf("bad siginfo code %d in pipe notification.\n", code); 431 #endif 432 break; 433 } 434 435 selnotify(&selp->pipe_sel, band, NOTE_SUBMIT); 436 437 if (sigp == NULL || (sigp->pipe_state & PIPE_ASYNC) == 0) 438 return; 439 440 fownsignal(sigp->pipe_pgid, SIGIO, code, band, selp); 441 } 442 443 static int 444 pipe_read(file_t *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 445 int flags) 446 { 447 struct pipe *rpipe = (struct pipe *) fp->f_data; 448 struct pipebuf *bp = &rpipe->pipe_buffer; 449 kmutex_t *lock = rpipe->pipe_lock; 450 int error; 451 size_t nread = 0; 452 size_t size; 453 size_t ocnt; 454 unsigned int wakeup_state = 0; 455 456 mutex_enter(lock); 457 ++rpipe->pipe_busy; 458 ocnt = bp->cnt; 459 460 again: 461 error = pipelock(rpipe, 1); 462 if (error) 463 goto unlocked_error; 464 465 while (uio->uio_resid) { 466 /* 467 * Normal pipe buffer receive. 468 */ 469 if (bp->cnt > 0) { 470 size = bp->size - bp->out; 471 if (size > bp->cnt) 472 size = bp->cnt; 473 if (size > uio->uio_resid) 474 size = uio->uio_resid; 475 476 mutex_exit(lock); 477 error = uiomove((char *)bp->buffer + bp->out, size, uio); 478 mutex_enter(lock); 479 if (error) 480 break; 481 482 bp->out += size; 483 if (bp->out >= bp->size) 484 bp->out = 0; 485 486 bp->cnt -= size; 487 488 /* 489 * If there is no more to read in the pipe, reset 490 * its pointers to the beginning. This improves 491 * cache hit stats. 492 */ 493 if (bp->cnt == 0) { 494 bp->in = 0; 495 bp->out = 0; 496 } 497 nread += size; 498 continue; 499 } 500 501 #ifndef PIPE_NODIRECT 502 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) { 503 /* 504 * Direct copy, bypassing a kernel buffer. 505 */ 506 void *va; 507 u_int gen; 508 509 KASSERT(rpipe->pipe_state & PIPE_DIRECTW); 510 511 size = rpipe->pipe_map.cnt; 512 if (size > uio->uio_resid) 513 size = uio->uio_resid; 514 515 va = (char *)rpipe->pipe_map.kva + rpipe->pipe_map.pos; 516 gen = rpipe->pipe_map.egen; 517 mutex_exit(lock); 518 519 /* 520 * Consume emap and read the data from loaned pages. 521 */ 522 uvm_emap_consume(gen); 523 error = uiomove(va, size, uio); 524 525 mutex_enter(lock); 526 if (error) 527 break; 528 nread += size; 529 rpipe->pipe_map.pos += size; 530 rpipe->pipe_map.cnt -= size; 531 if (rpipe->pipe_map.cnt == 0) { 532 rpipe->pipe_state &= ~PIPE_DIRECTR; 533 cv_broadcast(&rpipe->pipe_wcv); 534 } 535 continue; 536 } 537 #endif 538 /* 539 * Break if some data was read. 540 */ 541 if (nread > 0) 542 break; 543 544 /* 545 * Detect EOF condition. 546 * Read returns 0 on EOF, no need to set error. 547 */ 548 if (rpipe->pipe_state & PIPE_EOF) 549 break; 550 551 /* 552 * Don't block on non-blocking I/O. 553 */ 554 if (fp->f_flag & FNONBLOCK) { 555 error = EAGAIN; 556 break; 557 } 558 559 /* 560 * Unlock the pipe buffer for our remaining processing. 561 * We will either break out with an error or we will 562 * sleep and relock to loop. 563 */ 564 pipeunlock(rpipe); 565 566 /* 567 * Re-check to see if more direct writes are pending. 568 */ 569 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) 570 goto again; 571 572 #if 1 /* XXX (dsl) I'm sure these aren't needed here ... */ 573 /* 574 * We want to read more, wake up select/poll. 575 */ 576 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 577 578 /* 579 * If the "write-side" is blocked, wake it up now. 580 */ 581 cv_broadcast(&rpipe->pipe_wcv); 582 #endif 583 584 if (wakeup_state & PIPE_RESTART) { 585 error = ERESTART; 586 goto unlocked_error; 587 } 588 589 /* Now wait until the pipe is filled */ 590 error = cv_wait_sig(&rpipe->pipe_rcv, lock); 591 if (error != 0) 592 goto unlocked_error; 593 wakeup_state = rpipe->pipe_state; 594 goto again; 595 } 596 597 if (error == 0) 598 getnanotime(&rpipe->pipe_atime); 599 pipeunlock(rpipe); 600 601 unlocked_error: 602 --rpipe->pipe_busy; 603 if (rpipe->pipe_busy == 0) { 604 rpipe->pipe_state &= ~PIPE_RESTART; 605 cv_broadcast(&rpipe->pipe_draincv); 606 } 607 if (bp->cnt < MINPIPESIZE) { 608 cv_broadcast(&rpipe->pipe_wcv); 609 } 610 611 /* 612 * If anything was read off the buffer, signal to the writer it's 613 * possible to write more data. Also send signal if we are here for the 614 * first time after last write. 615 */ 616 if ((bp->size - bp->cnt) >= PIPE_BUF 617 && (ocnt != bp->cnt || (rpipe->pipe_state & PIPE_SIGNALR))) { 618 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 619 rpipe->pipe_state &= ~PIPE_SIGNALR; 620 } 621 622 mutex_exit(lock); 623 return (error); 624 } 625 626 #ifndef PIPE_NODIRECT 627 /* 628 * Allocate structure for loan transfer. 629 */ 630 static int 631 pipe_loan_alloc(struct pipe *wpipe, int npages) 632 { 633 vsize_t len; 634 635 len = (vsize_t)npages << PAGE_SHIFT; 636 atomic_add_int(&amountpipekva, len); 637 wpipe->pipe_map.kva = uvm_km_alloc(kernel_map, len, 0, 638 UVM_KMF_VAONLY | UVM_KMF_WAITVA); 639 if (wpipe->pipe_map.kva == 0) { 640 atomic_add_int(&amountpipekva, -len); 641 return (ENOMEM); 642 } 643 644 wpipe->pipe_map.npages = npages; 645 wpipe->pipe_map.pgs = kmem_alloc(npages * sizeof(struct vm_page *), 646 KM_SLEEP); 647 return (0); 648 } 649 650 /* 651 * Free resources allocated for loan transfer. 652 */ 653 static void 654 pipe_loan_free(struct pipe *wpipe) 655 { 656 vsize_t len; 657 658 len = (vsize_t)wpipe->pipe_map.npages << PAGE_SHIFT; 659 uvm_emap_remove(wpipe->pipe_map.kva, len); /* XXX */ 660 uvm_km_free(kernel_map, wpipe->pipe_map.kva, len, UVM_KMF_VAONLY); 661 wpipe->pipe_map.kva = 0; 662 atomic_add_int(&amountpipekva, -len); 663 kmem_free(wpipe->pipe_map.pgs, 664 wpipe->pipe_map.npages * sizeof(struct vm_page *)); 665 wpipe->pipe_map.pgs = NULL; 666 } 667 668 /* 669 * NetBSD direct write, using uvm_loan() mechanism. 670 * This implements the pipe buffer write mechanism. Note that only 671 * a direct write OR a normal pipe write can be pending at any given time. 672 * If there are any characters in the pipe buffer, the direct write will 673 * be deferred until the receiving process grabs all of the bytes from 674 * the pipe buffer. Then the direct mapping write is set-up. 675 * 676 * Called with the long-term pipe lock held. 677 */ 678 static int 679 pipe_direct_write(file_t *fp, struct pipe *wpipe, struct uio *uio) 680 { 681 struct vm_page **pgs; 682 vaddr_t bbase, base, bend; 683 vsize_t blen, bcnt; 684 int error, npages; 685 voff_t bpos; 686 kmutex_t *lock = wpipe->pipe_lock; 687 688 KASSERT(mutex_owned(wpipe->pipe_lock)); 689 KASSERT(wpipe->pipe_map.cnt == 0); 690 691 mutex_exit(lock); 692 693 /* 694 * Handle first PIPE_CHUNK_SIZE bytes of buffer. Deal with buffers 695 * not aligned to PAGE_SIZE. 696 */ 697 bbase = (vaddr_t)uio->uio_iov->iov_base; 698 base = trunc_page(bbase); 699 bend = round_page(bbase + uio->uio_iov->iov_len); 700 blen = bend - base; 701 bpos = bbase - base; 702 703 if (blen > PIPE_DIRECT_CHUNK) { 704 blen = PIPE_DIRECT_CHUNK; 705 bend = base + blen; 706 bcnt = PIPE_DIRECT_CHUNK - bpos; 707 } else { 708 bcnt = uio->uio_iov->iov_len; 709 } 710 npages = blen >> PAGE_SHIFT; 711 712 /* 713 * Free the old kva if we need more pages than we have 714 * allocated. 715 */ 716 if (wpipe->pipe_map.kva != 0 && npages > wpipe->pipe_map.npages) 717 pipe_loan_free(wpipe); 718 719 /* Allocate new kva. */ 720 if (wpipe->pipe_map.kva == 0) { 721 error = pipe_loan_alloc(wpipe, npages); 722 if (error) { 723 mutex_enter(lock); 724 return (error); 725 } 726 } 727 728 /* Loan the write buffer memory from writer process */ 729 pgs = wpipe->pipe_map.pgs; 730 error = uvm_loan(&uio->uio_vmspace->vm_map, base, blen, 731 pgs, UVM_LOAN_TOPAGE); 732 if (error) { 733 pipe_loan_free(wpipe); 734 mutex_enter(lock); 735 return (ENOMEM); /* so that caller fallback to ordinary write */ 736 } 737 738 /* Enter the loaned pages to KVA, produce new emap generation number. */ 739 uvm_emap_enter(wpipe->pipe_map.kva, pgs, npages); 740 wpipe->pipe_map.egen = uvm_emap_produce(); 741 742 /* Now we can put the pipe in direct write mode */ 743 wpipe->pipe_map.pos = bpos; 744 wpipe->pipe_map.cnt = bcnt; 745 746 /* 747 * But before we can let someone do a direct read, we 748 * have to wait until the pipe is drained. Release the 749 * pipe lock while we wait. 750 */ 751 mutex_enter(lock); 752 wpipe->pipe_state |= PIPE_DIRECTW; 753 pipeunlock(wpipe); 754 755 while (error == 0 && wpipe->pipe_buffer.cnt > 0) { 756 cv_broadcast(&wpipe->pipe_rcv); 757 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 758 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 759 error = EPIPE; 760 } 761 762 /* Pipe is drained; next read will off the direct buffer */ 763 wpipe->pipe_state |= PIPE_DIRECTR; 764 765 /* Wait until the reader is done */ 766 while (error == 0 && (wpipe->pipe_state & PIPE_DIRECTR)) { 767 cv_broadcast(&wpipe->pipe_rcv); 768 pipeselwakeup(wpipe, wpipe, POLL_IN); 769 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 770 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 771 error = EPIPE; 772 } 773 774 /* Take pipe out of direct write mode */ 775 wpipe->pipe_state &= ~(PIPE_DIRECTW | PIPE_DIRECTR); 776 777 /* Acquire the pipe lock and cleanup */ 778 (void)pipelock(wpipe, 0); 779 mutex_exit(lock); 780 781 if (pgs != NULL) { 782 /* XXX: uvm_emap_remove */ 783 uvm_unloan(pgs, npages, UVM_LOAN_TOPAGE); 784 } 785 if (error || amountpipekva > maxpipekva) 786 pipe_loan_free(wpipe); 787 788 mutex_enter(lock); 789 if (error) { 790 pipeselwakeup(wpipe, wpipe, POLL_ERR); 791 792 /* 793 * If nothing was read from what we offered, return error 794 * straight on. Otherwise update uio resid first. Caller 795 * will deal with the error condition, returning short 796 * write, error, or restarting the write(2) as appropriate. 797 */ 798 if (wpipe->pipe_map.cnt == bcnt) { 799 wpipe->pipe_map.cnt = 0; 800 cv_broadcast(&wpipe->pipe_wcv); 801 return (error); 802 } 803 804 bcnt -= wpipe->pipe_map.cnt; 805 } 806 807 uio->uio_resid -= bcnt; 808 /* uio_offset not updated, not set/used for write(2) */ 809 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + bcnt; 810 uio->uio_iov->iov_len -= bcnt; 811 if (uio->uio_iov->iov_len == 0) { 812 uio->uio_iov++; 813 uio->uio_iovcnt--; 814 } 815 816 wpipe->pipe_map.cnt = 0; 817 return (error); 818 } 819 #endif /* !PIPE_NODIRECT */ 820 821 static int 822 pipe_write(file_t *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 823 int flags) 824 { 825 struct pipe *wpipe, *rpipe; 826 struct pipebuf *bp; 827 kmutex_t *lock; 828 int error; 829 unsigned int wakeup_state = 0; 830 831 /* We want to write to our peer */ 832 rpipe = (struct pipe *) fp->f_data; 833 lock = rpipe->pipe_lock; 834 error = 0; 835 836 mutex_enter(lock); 837 wpipe = rpipe->pipe_peer; 838 839 /* 840 * Detect loss of pipe read side, issue SIGPIPE if lost. 841 */ 842 if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF) != 0) { 843 mutex_exit(lock); 844 return EPIPE; 845 } 846 ++wpipe->pipe_busy; 847 848 /* Aquire the long-term pipe lock */ 849 if ((error = pipelock(wpipe, 1)) != 0) { 850 --wpipe->pipe_busy; 851 if (wpipe->pipe_busy == 0) { 852 wpipe->pipe_state &= ~PIPE_RESTART; 853 cv_broadcast(&wpipe->pipe_draincv); 854 } 855 mutex_exit(lock); 856 return (error); 857 } 858 859 bp = &wpipe->pipe_buffer; 860 861 /* 862 * If it is advantageous to resize the pipe buffer, do so. 863 */ 864 if ((uio->uio_resid > PIPE_SIZE) && 865 (nbigpipe < maxbigpipes) && 866 #ifndef PIPE_NODIRECT 867 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 868 #endif 869 (bp->size <= PIPE_SIZE) && (bp->cnt == 0)) { 870 871 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) 872 atomic_inc_uint(&nbigpipe); 873 } 874 875 while (uio->uio_resid) { 876 size_t space; 877 878 #ifndef PIPE_NODIRECT 879 /* 880 * Pipe buffered writes cannot be coincidental with 881 * direct writes. Also, only one direct write can be 882 * in progress at any one time. We wait until the currently 883 * executing direct write is completed before continuing. 884 * 885 * We break out if a signal occurs or the reader goes away. 886 */ 887 while (error == 0 && wpipe->pipe_state & PIPE_DIRECTW) { 888 cv_broadcast(&wpipe->pipe_rcv); 889 pipeunlock(wpipe); 890 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 891 (void)pipelock(wpipe, 0); 892 if (wpipe->pipe_state & PIPE_EOF) 893 error = EPIPE; 894 } 895 if (error) 896 break; 897 898 /* 899 * If the transfer is large, we can gain performance if 900 * we do process-to-process copies directly. 901 * If the write is non-blocking, we don't use the 902 * direct write mechanism. 903 * 904 * The direct write mechanism will detect the reader going 905 * away on us. 906 */ 907 if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) && 908 (fp->f_flag & FNONBLOCK) == 0 && 909 (wpipe->pipe_map.kva || (amountpipekva < limitpipekva))) { 910 error = pipe_direct_write(fp, wpipe, uio); 911 912 /* 913 * Break out if error occurred, unless it's ENOMEM. 914 * ENOMEM means we failed to allocate some resources 915 * for direct write, so we just fallback to ordinary 916 * write. If the direct write was successful, 917 * process rest of data via ordinary write. 918 */ 919 if (error == 0) 920 continue; 921 922 if (error != ENOMEM) 923 break; 924 } 925 #endif /* PIPE_NODIRECT */ 926 927 space = bp->size - bp->cnt; 928 929 /* Writes of size <= PIPE_BUF must be atomic. */ 930 if ((space < uio->uio_resid) && (uio->uio_resid <= PIPE_BUF)) 931 space = 0; 932 933 if (space > 0) { 934 int size; /* Transfer size */ 935 int segsize; /* first segment to transfer */ 936 937 /* 938 * Transfer size is minimum of uio transfer 939 * and free space in pipe buffer. 940 */ 941 if (space > uio->uio_resid) 942 size = uio->uio_resid; 943 else 944 size = space; 945 /* 946 * First segment to transfer is minimum of 947 * transfer size and contiguous space in 948 * pipe buffer. If first segment to transfer 949 * is less than the transfer size, we've got 950 * a wraparound in the buffer. 951 */ 952 segsize = bp->size - bp->in; 953 if (segsize > size) 954 segsize = size; 955 956 /* Transfer first segment */ 957 mutex_exit(lock); 958 error = uiomove((char *)bp->buffer + bp->in, segsize, 959 uio); 960 961 if (error == 0 && segsize < size) { 962 /* 963 * Transfer remaining part now, to 964 * support atomic writes. Wraparound 965 * happened. 966 */ 967 KASSERT(bp->in + segsize == bp->size); 968 error = uiomove(bp->buffer, 969 size - segsize, uio); 970 } 971 mutex_enter(lock); 972 if (error) 973 break; 974 975 bp->in += size; 976 if (bp->in >= bp->size) { 977 KASSERT(bp->in == size - segsize + bp->size); 978 bp->in = size - segsize; 979 } 980 981 bp->cnt += size; 982 KASSERT(bp->cnt <= bp->size); 983 wakeup_state = 0; 984 } else { 985 /* 986 * If the "read-side" has been blocked, wake it up now. 987 */ 988 cv_broadcast(&wpipe->pipe_rcv); 989 990 /* 991 * Don't block on non-blocking I/O. 992 */ 993 if (fp->f_flag & FNONBLOCK) { 994 error = EAGAIN; 995 break; 996 } 997 998 /* 999 * We have no more space and have something to offer, 1000 * wake up select/poll. 1001 */ 1002 if (bp->cnt) 1003 pipeselwakeup(wpipe, wpipe, POLL_IN); 1004 1005 if (wakeup_state & PIPE_RESTART) { 1006 error = ERESTART; 1007 break; 1008 } 1009 1010 pipeunlock(wpipe); 1011 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 1012 (void)pipelock(wpipe, 0); 1013 if (error != 0) 1014 break; 1015 /* 1016 * If read side wants to go away, we just issue a signal 1017 * to ourselves. 1018 */ 1019 if (wpipe->pipe_state & PIPE_EOF) { 1020 error = EPIPE; 1021 break; 1022 } 1023 wakeup_state = wpipe->pipe_state; 1024 } 1025 } 1026 1027 --wpipe->pipe_busy; 1028 if (wpipe->pipe_busy == 0) { 1029 wpipe->pipe_state &= ~PIPE_RESTART; 1030 cv_broadcast(&wpipe->pipe_draincv); 1031 } 1032 if (bp->cnt > 0) { 1033 cv_broadcast(&wpipe->pipe_rcv); 1034 } 1035 1036 /* 1037 * Don't return EPIPE if I/O was successful 1038 */ 1039 if (error == EPIPE && bp->cnt == 0 && uio->uio_resid == 0) 1040 error = 0; 1041 1042 if (error == 0) 1043 getnanotime(&wpipe->pipe_mtime); 1044 1045 /* 1046 * We have something to offer, wake up select/poll. 1047 * wpipe->pipe_map.cnt is always 0 in this point (direct write 1048 * is only done synchronously), so check only wpipe->pipe_buffer.cnt 1049 */ 1050 if (bp->cnt) 1051 pipeselwakeup(wpipe, wpipe, POLL_IN); 1052 1053 /* 1054 * Arrange for next read(2) to do a signal. 1055 */ 1056 wpipe->pipe_state |= PIPE_SIGNALR; 1057 1058 pipeunlock(wpipe); 1059 mutex_exit(lock); 1060 return (error); 1061 } 1062 1063 /* 1064 * We implement a very minimal set of ioctls for compatibility with sockets. 1065 */ 1066 int 1067 pipe_ioctl(file_t *fp, u_long cmd, void *data) 1068 { 1069 struct pipe *pipe = fp->f_data; 1070 kmutex_t *lock = pipe->pipe_lock; 1071 1072 switch (cmd) { 1073 1074 case FIONBIO: 1075 return (0); 1076 1077 case FIOASYNC: 1078 mutex_enter(lock); 1079 if (*(int *)data) { 1080 pipe->pipe_state |= PIPE_ASYNC; 1081 } else { 1082 pipe->pipe_state &= ~PIPE_ASYNC; 1083 } 1084 mutex_exit(lock); 1085 return (0); 1086 1087 case FIONREAD: 1088 mutex_enter(lock); 1089 #ifndef PIPE_NODIRECT 1090 if (pipe->pipe_state & PIPE_DIRECTW) 1091 *(int *)data = pipe->pipe_map.cnt; 1092 else 1093 #endif 1094 *(int *)data = pipe->pipe_buffer.cnt; 1095 mutex_exit(lock); 1096 return (0); 1097 1098 case FIONWRITE: 1099 /* Look at other side */ 1100 pipe = pipe->pipe_peer; 1101 mutex_enter(lock); 1102 #ifndef PIPE_NODIRECT 1103 if (pipe->pipe_state & PIPE_DIRECTW) 1104 *(int *)data = pipe->pipe_map.cnt; 1105 else 1106 #endif 1107 *(int *)data = pipe->pipe_buffer.cnt; 1108 mutex_exit(lock); 1109 return (0); 1110 1111 case FIONSPACE: 1112 /* Look at other side */ 1113 pipe = pipe->pipe_peer; 1114 mutex_enter(lock); 1115 #ifndef PIPE_NODIRECT 1116 /* 1117 * If we're in direct-mode, we don't really have a 1118 * send queue, and any other write will block. Thus 1119 * zero seems like the best answer. 1120 */ 1121 if (pipe->pipe_state & PIPE_DIRECTW) 1122 *(int *)data = 0; 1123 else 1124 #endif 1125 *(int *)data = pipe->pipe_buffer.size - 1126 pipe->pipe_buffer.cnt; 1127 mutex_exit(lock); 1128 return (0); 1129 1130 case TIOCSPGRP: 1131 case FIOSETOWN: 1132 return fsetown(&pipe->pipe_pgid, cmd, data); 1133 1134 case TIOCGPGRP: 1135 case FIOGETOWN: 1136 return fgetown(pipe->pipe_pgid, cmd, data); 1137 1138 } 1139 return (EPASSTHROUGH); 1140 } 1141 1142 int 1143 pipe_poll(file_t *fp, int events) 1144 { 1145 struct pipe *rpipe = fp->f_data; 1146 struct pipe *wpipe; 1147 int eof = 0; 1148 int revents = 0; 1149 1150 mutex_enter(rpipe->pipe_lock); 1151 wpipe = rpipe->pipe_peer; 1152 1153 if (events & (POLLIN | POLLRDNORM)) 1154 if ((rpipe->pipe_buffer.cnt > 0) || 1155 #ifndef PIPE_NODIRECT 1156 (rpipe->pipe_state & PIPE_DIRECTR) || 1157 #endif 1158 (rpipe->pipe_state & PIPE_EOF)) 1159 revents |= events & (POLLIN | POLLRDNORM); 1160 1161 eof |= (rpipe->pipe_state & PIPE_EOF); 1162 1163 if (wpipe == NULL) 1164 revents |= events & (POLLOUT | POLLWRNORM); 1165 else { 1166 if (events & (POLLOUT | POLLWRNORM)) 1167 if ((wpipe->pipe_state & PIPE_EOF) || ( 1168 #ifndef PIPE_NODIRECT 1169 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 1170 #endif 1171 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 1172 revents |= events & (POLLOUT | POLLWRNORM); 1173 1174 eof |= (wpipe->pipe_state & PIPE_EOF); 1175 } 1176 1177 if (wpipe == NULL || eof) 1178 revents |= POLLHUP; 1179 1180 if (revents == 0) { 1181 if (events & (POLLIN | POLLRDNORM)) 1182 selrecord(curlwp, &rpipe->pipe_sel); 1183 1184 if (events & (POLLOUT | POLLWRNORM)) 1185 selrecord(curlwp, &wpipe->pipe_sel); 1186 } 1187 mutex_exit(rpipe->pipe_lock); 1188 1189 return (revents); 1190 } 1191 1192 static int 1193 pipe_stat(file_t *fp, struct stat *ub) 1194 { 1195 struct pipe *pipe = fp->f_data; 1196 1197 mutex_enter(pipe->pipe_lock); 1198 memset(ub, 0, sizeof(*ub)); 1199 ub->st_mode = S_IFIFO | S_IRUSR | S_IWUSR; 1200 ub->st_blksize = pipe->pipe_buffer.size; 1201 if (ub->st_blksize == 0 && pipe->pipe_peer) 1202 ub->st_blksize = pipe->pipe_peer->pipe_buffer.size; 1203 ub->st_size = pipe->pipe_buffer.cnt; 1204 ub->st_blocks = (ub->st_size) ? 1 : 0; 1205 ub->st_atimespec = pipe->pipe_atime; 1206 ub->st_mtimespec = pipe->pipe_mtime; 1207 ub->st_ctimespec = ub->st_birthtimespec = pipe->pipe_btime; 1208 ub->st_uid = kauth_cred_geteuid(fp->f_cred); 1209 ub->st_gid = kauth_cred_getegid(fp->f_cred); 1210 1211 /* 1212 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 1213 * XXX (st_dev, st_ino) should be unique. 1214 */ 1215 mutex_exit(pipe->pipe_lock); 1216 return 0; 1217 } 1218 1219 static int 1220 pipe_close(file_t *fp) 1221 { 1222 struct pipe *pipe = fp->f_data; 1223 1224 fp->f_data = NULL; 1225 pipeclose(pipe); 1226 return (0); 1227 } 1228 1229 static void 1230 pipe_restart(file_t *fp) 1231 { 1232 struct pipe *pipe = fp->f_data; 1233 1234 /* 1235 * Unblock blocked reads/writes in order to allow close() to complete. 1236 * System calls return ERESTART so that the fd is revalidated. 1237 * (Partial writes return the transfer length.) 1238 */ 1239 mutex_enter(pipe->pipe_lock); 1240 pipe->pipe_state |= PIPE_RESTART; 1241 /* Wakeup both cvs, maybe we only need one, but maybe there are some 1242 * other paths where wakeup is needed, and it saves deciding which! */ 1243 cv_broadcast(&pipe->pipe_rcv); 1244 cv_broadcast(&pipe->pipe_wcv); 1245 mutex_exit(pipe->pipe_lock); 1246 } 1247 1248 static void 1249 pipe_free_kmem(struct pipe *pipe) 1250 { 1251 1252 if (pipe->pipe_buffer.buffer != NULL) { 1253 if (pipe->pipe_buffer.size > PIPE_SIZE) { 1254 atomic_dec_uint(&nbigpipe); 1255 } 1256 if (pipe->pipe_buffer.buffer != (void *)pipe->pipe_kmem) { 1257 uvm_km_free(kernel_map, 1258 (vaddr_t)pipe->pipe_buffer.buffer, 1259 pipe->pipe_buffer.size, UVM_KMF_PAGEABLE); 1260 atomic_add_int(&amountpipekva, 1261 -pipe->pipe_buffer.size); 1262 } 1263 pipe->pipe_buffer.buffer = NULL; 1264 } 1265 #ifndef PIPE_NODIRECT 1266 if (pipe->pipe_map.kva != 0) { 1267 pipe_loan_free(pipe); 1268 pipe->pipe_map.cnt = 0; 1269 pipe->pipe_map.kva = 0; 1270 pipe->pipe_map.pos = 0; 1271 pipe->pipe_map.npages = 0; 1272 } 1273 #endif /* !PIPE_NODIRECT */ 1274 } 1275 1276 /* 1277 * Shutdown the pipe. 1278 */ 1279 static void 1280 pipeclose(struct pipe *pipe) 1281 { 1282 kmutex_t *lock; 1283 struct pipe *ppipe; 1284 1285 if (pipe == NULL) 1286 return; 1287 1288 KASSERT(cv_is_valid(&pipe->pipe_rcv)); 1289 KASSERT(cv_is_valid(&pipe->pipe_wcv)); 1290 KASSERT(cv_is_valid(&pipe->pipe_draincv)); 1291 KASSERT(cv_is_valid(&pipe->pipe_lkcv)); 1292 1293 lock = pipe->pipe_lock; 1294 if (lock == NULL) 1295 /* Must have failed during create */ 1296 goto free_resources; 1297 1298 mutex_enter(lock); 1299 pipeselwakeup(pipe, pipe, POLL_HUP); 1300 1301 /* 1302 * If the other side is blocked, wake it up saying that 1303 * we want to close it down. 1304 */ 1305 pipe->pipe_state |= PIPE_EOF; 1306 if (pipe->pipe_busy) { 1307 while (pipe->pipe_busy) { 1308 cv_broadcast(&pipe->pipe_wcv); 1309 cv_wait_sig(&pipe->pipe_draincv, lock); 1310 } 1311 } 1312 1313 /* 1314 * Disconnect from peer. 1315 */ 1316 if ((ppipe = pipe->pipe_peer) != NULL) { 1317 pipeselwakeup(ppipe, ppipe, POLL_HUP); 1318 ppipe->pipe_state |= PIPE_EOF; 1319 cv_broadcast(&ppipe->pipe_rcv); 1320 ppipe->pipe_peer = NULL; 1321 } 1322 1323 /* 1324 * Any knote objects still left in the list are 1325 * the one attached by peer. Since no one will 1326 * traverse this list, we just clear it. 1327 */ 1328 SLIST_INIT(&pipe->pipe_sel.sel_klist); 1329 1330 KASSERT((pipe->pipe_state & PIPE_LOCKFL) == 0); 1331 mutex_exit(lock); 1332 mutex_obj_free(lock); 1333 1334 /* 1335 * Free resources. 1336 */ 1337 free_resources: 1338 pipe->pipe_pgid = 0; 1339 pipe->pipe_state = PIPE_SIGNALR; 1340 pipe_free_kmem(pipe); 1341 if (pipe->pipe_kmem != 0) { 1342 pool_cache_put(pipe_rd_cache, pipe); 1343 } else { 1344 pool_cache_put(pipe_wr_cache, pipe); 1345 } 1346 } 1347 1348 static void 1349 filt_pipedetach(struct knote *kn) 1350 { 1351 struct pipe *pipe; 1352 kmutex_t *lock; 1353 1354 pipe = ((file_t *)kn->kn_obj)->f_data; 1355 lock = pipe->pipe_lock; 1356 1357 mutex_enter(lock); 1358 1359 switch(kn->kn_filter) { 1360 case EVFILT_WRITE: 1361 /* Need the peer structure, not our own. */ 1362 pipe = pipe->pipe_peer; 1363 1364 /* If reader end already closed, just return. */ 1365 if (pipe == NULL) { 1366 mutex_exit(lock); 1367 return; 1368 } 1369 1370 break; 1371 default: 1372 /* Nothing to do. */ 1373 break; 1374 } 1375 1376 KASSERT(kn->kn_hook == pipe); 1377 SLIST_REMOVE(&pipe->pipe_sel.sel_klist, kn, knote, kn_selnext); 1378 mutex_exit(lock); 1379 } 1380 1381 static int 1382 filt_piperead(struct knote *kn, long hint) 1383 { 1384 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_data; 1385 struct pipe *wpipe; 1386 1387 if ((hint & NOTE_SUBMIT) == 0) { 1388 mutex_enter(rpipe->pipe_lock); 1389 } 1390 wpipe = rpipe->pipe_peer; 1391 kn->kn_data = rpipe->pipe_buffer.cnt; 1392 1393 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1394 kn->kn_data = rpipe->pipe_map.cnt; 1395 1396 if ((rpipe->pipe_state & PIPE_EOF) || 1397 (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1398 kn->kn_flags |= EV_EOF; 1399 if ((hint & NOTE_SUBMIT) == 0) { 1400 mutex_exit(rpipe->pipe_lock); 1401 } 1402 return (1); 1403 } 1404 1405 if ((hint & NOTE_SUBMIT) == 0) { 1406 mutex_exit(rpipe->pipe_lock); 1407 } 1408 return (kn->kn_data > 0); 1409 } 1410 1411 static int 1412 filt_pipewrite(struct knote *kn, long hint) 1413 { 1414 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_data; 1415 struct pipe *wpipe; 1416 1417 if ((hint & NOTE_SUBMIT) == 0) { 1418 mutex_enter(rpipe->pipe_lock); 1419 } 1420 wpipe = rpipe->pipe_peer; 1421 1422 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1423 kn->kn_data = 0; 1424 kn->kn_flags |= EV_EOF; 1425 if ((hint & NOTE_SUBMIT) == 0) { 1426 mutex_exit(rpipe->pipe_lock); 1427 } 1428 return (1); 1429 } 1430 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1431 if (wpipe->pipe_state & PIPE_DIRECTW) 1432 kn->kn_data = 0; 1433 1434 if ((hint & NOTE_SUBMIT) == 0) { 1435 mutex_exit(rpipe->pipe_lock); 1436 } 1437 return (kn->kn_data >= PIPE_BUF); 1438 } 1439 1440 static const struct filterops pipe_rfiltops = 1441 { 1, NULL, filt_pipedetach, filt_piperead }; 1442 static const struct filterops pipe_wfiltops = 1443 { 1, NULL, filt_pipedetach, filt_pipewrite }; 1444 1445 static int 1446 pipe_kqfilter(file_t *fp, struct knote *kn) 1447 { 1448 struct pipe *pipe; 1449 kmutex_t *lock; 1450 1451 pipe = ((file_t *)kn->kn_obj)->f_data; 1452 lock = pipe->pipe_lock; 1453 1454 mutex_enter(lock); 1455 1456 switch (kn->kn_filter) { 1457 case EVFILT_READ: 1458 kn->kn_fop = &pipe_rfiltops; 1459 break; 1460 case EVFILT_WRITE: 1461 kn->kn_fop = &pipe_wfiltops; 1462 pipe = pipe->pipe_peer; 1463 if (pipe == NULL) { 1464 /* Other end of pipe has been closed. */ 1465 mutex_exit(lock); 1466 return (EBADF); 1467 } 1468 break; 1469 default: 1470 mutex_exit(lock); 1471 return (EINVAL); 1472 } 1473 1474 kn->kn_hook = pipe; 1475 SLIST_INSERT_HEAD(&pipe->pipe_sel.sel_klist, kn, kn_selnext); 1476 mutex_exit(lock); 1477 1478 return (0); 1479 } 1480 1481 /* 1482 * Handle pipe sysctls. 1483 */ 1484 SYSCTL_SETUP(sysctl_kern_pipe_setup, "sysctl kern.pipe subtree setup") 1485 { 1486 1487 sysctl_createv(clog, 0, NULL, NULL, 1488 CTLFLAG_PERMANENT, 1489 CTLTYPE_NODE, "kern", NULL, 1490 NULL, 0, NULL, 0, 1491 CTL_KERN, CTL_EOL); 1492 sysctl_createv(clog, 0, NULL, NULL, 1493 CTLFLAG_PERMANENT, 1494 CTLTYPE_NODE, "pipe", 1495 SYSCTL_DESCR("Pipe settings"), 1496 NULL, 0, NULL, 0, 1497 CTL_KERN, KERN_PIPE, CTL_EOL); 1498 1499 sysctl_createv(clog, 0, NULL, NULL, 1500 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1501 CTLTYPE_INT, "maxkvasz", 1502 SYSCTL_DESCR("Maximum amount of kernel memory to be " 1503 "used for pipes"), 1504 NULL, 0, &maxpipekva, 0, 1505 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXKVASZ, CTL_EOL); 1506 sysctl_createv(clog, 0, NULL, NULL, 1507 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1508 CTLTYPE_INT, "maxloankvasz", 1509 SYSCTL_DESCR("Limit for direct transfers via page loan"), 1510 NULL, 0, &limitpipekva, 0, 1511 CTL_KERN, KERN_PIPE, KERN_PIPE_LIMITKVA, CTL_EOL); 1512 sysctl_createv(clog, 0, NULL, NULL, 1513 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1514 CTLTYPE_INT, "maxbigpipes", 1515 SYSCTL_DESCR("Maximum number of \"big\" pipes"), 1516 NULL, 0, &maxbigpipes, 0, 1517 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXBIGPIPES, CTL_EOL); 1518 sysctl_createv(clog, 0, NULL, NULL, 1519 CTLFLAG_PERMANENT, 1520 CTLTYPE_INT, "nbigpipes", 1521 SYSCTL_DESCR("Number of \"big\" pipes"), 1522 NULL, 0, &nbigpipe, 0, 1523 CTL_KERN, KERN_PIPE, KERN_PIPE_NBIGPIPES, CTL_EOL); 1524 sysctl_createv(clog, 0, NULL, NULL, 1525 CTLFLAG_PERMANENT, 1526 CTLTYPE_INT, "kvasize", 1527 SYSCTL_DESCR("Amount of kernel memory consumed by pipe " 1528 "buffers"), 1529 NULL, 0, &amountpipekva, 0, 1530 CTL_KERN, KERN_PIPE, KERN_PIPE_KVASIZE, CTL_EOL); 1531 } 1532