1 /* $NetBSD: sys_pipe.c,v 1.117 2009/07/15 21:09:41 rmind Exp $ */ 2 3 /*- 4 * Copyright (c) 2003, 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Paul Kranenburg, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1996 John S. Dyson 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice immediately at the beginning of the file, without modification, 41 * this list of conditions, and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. Absolutely no warranty of function or purpose is made by the author 46 * John S. Dyson. 47 * 4. Modifications may be freely made to this file if the above conditions 48 * are met. 49 */ 50 51 /* 52 * This file contains a high-performance replacement for the socket-based 53 * pipes scheme originally used. It does not support all features of 54 * sockets, but does do everything that pipes normally do. 55 * 56 * This code has two modes of operation, a small write mode and a large 57 * write mode. The small write mode acts like conventional pipes with 58 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 59 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 60 * and PIPE_SIZE in size it is mapped read-only into the kernel address space 61 * using the UVM page loan facility from where the receiving process can copy 62 * the data directly from the pages in the sending process. 63 * 64 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 65 * happen for small transfers so that the system will not spend all of 66 * its time context switching. PIPE_SIZE is constrained by the 67 * amount of kernel virtual memory. 68 */ 69 70 #include <sys/cdefs.h> 71 __KERNEL_RCSID(0, "$NetBSD: sys_pipe.c,v 1.117 2009/07/15 21:09:41 rmind Exp $"); 72 73 #include <sys/param.h> 74 #include <sys/systm.h> 75 #include <sys/proc.h> 76 #include <sys/fcntl.h> 77 #include <sys/file.h> 78 #include <sys/filedesc.h> 79 #include <sys/filio.h> 80 #include <sys/kernel.h> 81 #include <sys/ttycom.h> 82 #include <sys/stat.h> 83 #include <sys/poll.h> 84 #include <sys/signalvar.h> 85 #include <sys/vnode.h> 86 #include <sys/uio.h> 87 #include <sys/select.h> 88 #include <sys/mount.h> 89 #include <sys/syscallargs.h> 90 #include <sys/sysctl.h> 91 #include <sys/kauth.h> 92 #include <sys/atomic.h> 93 #include <sys/pipe.h> 94 95 #include <uvm/uvm.h> 96 97 /* 98 * Use this to disable direct I/O and decrease the code size: 99 * #define PIPE_NODIRECT 100 */ 101 102 /* XXX Disabled for now; rare hangs switching between direct/buffered */ 103 #define PIPE_NODIRECT 104 105 static int pipe_read(file_t *, off_t *, struct uio *, kauth_cred_t, int); 106 static int pipe_write(file_t *, off_t *, struct uio *, kauth_cred_t, int); 107 static int pipe_close(file_t *); 108 static int pipe_poll(file_t *, int); 109 static int pipe_kqfilter(file_t *, struct knote *); 110 static int pipe_stat(file_t *, struct stat *); 111 static int pipe_ioctl(file_t *, u_long, void *); 112 113 static const struct fileops pipeops = { 114 .fo_read = pipe_read, 115 .fo_write = pipe_write, 116 .fo_ioctl = pipe_ioctl, 117 .fo_fcntl = fnullop_fcntl, 118 .fo_poll = pipe_poll, 119 .fo_stat = pipe_stat, 120 .fo_close = pipe_close, 121 .fo_kqfilter = pipe_kqfilter, 122 .fo_drain = fnullop_drain, 123 }; 124 125 /* 126 * Default pipe buffer size(s), this can be kind-of large now because pipe 127 * space is pageable. The pipe code will try to maintain locality of 128 * reference for performance reasons, so small amounts of outstanding I/O 129 * will not wipe the cache. 130 */ 131 #define MINPIPESIZE (PIPE_SIZE / 3) 132 #define MAXPIPESIZE (2 * PIPE_SIZE / 3) 133 134 /* 135 * Maximum amount of kva for pipes -- this is kind-of a soft limit, but 136 * is there so that on large systems, we don't exhaust it. 137 */ 138 #define MAXPIPEKVA (8 * 1024 * 1024) 139 static u_int maxpipekva = MAXPIPEKVA; 140 141 /* 142 * Limit for direct transfers, we cannot, of course limit 143 * the amount of kva for pipes in general though. 144 */ 145 #define LIMITPIPEKVA (16 * 1024 * 1024) 146 static u_int limitpipekva = LIMITPIPEKVA; 147 148 /* 149 * Limit the number of "big" pipes 150 */ 151 #define LIMITBIGPIPES 32 152 static u_int maxbigpipes = LIMITBIGPIPES; 153 static u_int nbigpipe = 0; 154 155 /* 156 * Amount of KVA consumed by pipe buffers. 157 */ 158 static u_int amountpipekva = 0; 159 160 static void pipeclose(file_t *, struct pipe *); 161 static void pipe_free_kmem(struct pipe *); 162 static int pipe_create(struct pipe **, pool_cache_t, kmutex_t *); 163 static int pipelock(struct pipe *, int); 164 static inline void pipeunlock(struct pipe *); 165 static void pipeselwakeup(struct pipe *, struct pipe *, int); 166 #ifndef PIPE_NODIRECT 167 static int pipe_direct_write(file_t *, struct pipe *, struct uio *); 168 #endif 169 static int pipespace(struct pipe *, int); 170 static int pipe_ctor(void *, void *, int); 171 static void pipe_dtor(void *, void *); 172 173 #ifndef PIPE_NODIRECT 174 static int pipe_loan_alloc(struct pipe *, int); 175 static void pipe_loan_free(struct pipe *); 176 #endif /* PIPE_NODIRECT */ 177 178 static pool_cache_t pipe_wr_cache; 179 static pool_cache_t pipe_rd_cache; 180 181 void 182 pipe_init(void) 183 { 184 185 /* Writer side is not automatically allocated KVA. */ 186 pipe_wr_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "pipewr", 187 NULL, IPL_NONE, pipe_ctor, pipe_dtor, NULL); 188 KASSERT(pipe_wr_cache != NULL); 189 190 /* Reader side gets preallocated KVA. */ 191 pipe_rd_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "piperd", 192 NULL, IPL_NONE, pipe_ctor, pipe_dtor, (void *)1); 193 KASSERT(pipe_rd_cache != NULL); 194 } 195 196 static int 197 pipe_ctor(void *arg, void *obj, int flags) 198 { 199 struct pipe *pipe; 200 vaddr_t va; 201 202 pipe = obj; 203 204 memset(pipe, 0, sizeof(struct pipe)); 205 if (arg != NULL) { 206 /* Preallocate space. */ 207 va = uvm_km_alloc(kernel_map, PIPE_SIZE, 0, 208 UVM_KMF_PAGEABLE | UVM_KMF_WAITVA); 209 KASSERT(va != 0); 210 pipe->pipe_kmem = va; 211 atomic_add_int(&amountpipekva, PIPE_SIZE); 212 } 213 cv_init(&pipe->pipe_rcv, "piperd"); 214 cv_init(&pipe->pipe_wcv, "pipewr"); 215 cv_init(&pipe->pipe_draincv, "pipedrain"); 216 cv_init(&pipe->pipe_lkcv, "pipelk"); 217 selinit(&pipe->pipe_sel); 218 pipe->pipe_state = PIPE_SIGNALR; 219 220 return 0; 221 } 222 223 static void 224 pipe_dtor(void *arg, void *obj) 225 { 226 struct pipe *pipe; 227 228 pipe = obj; 229 230 cv_destroy(&pipe->pipe_rcv); 231 cv_destroy(&pipe->pipe_wcv); 232 cv_destroy(&pipe->pipe_draincv); 233 cv_destroy(&pipe->pipe_lkcv); 234 seldestroy(&pipe->pipe_sel); 235 if (pipe->pipe_kmem != 0) { 236 uvm_km_free(kernel_map, pipe->pipe_kmem, PIPE_SIZE, 237 UVM_KMF_PAGEABLE); 238 atomic_add_int(&amountpipekva, -PIPE_SIZE); 239 } 240 } 241 242 /* 243 * The pipe system call for the DTYPE_PIPE type of pipes 244 */ 245 int 246 sys_pipe(struct lwp *l, const void *v, register_t *retval) 247 { 248 struct pipe *rpipe, *wpipe; 249 file_t *rf, *wf; 250 kmutex_t *mutex; 251 int fd, error; 252 proc_t *p; 253 254 p = curproc; 255 rpipe = wpipe = NULL; 256 mutex = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 257 if (mutex == NULL) 258 return (ENOMEM); 259 mutex_obj_hold(mutex); 260 if (pipe_create(&rpipe, pipe_rd_cache, mutex) || 261 pipe_create(&wpipe, pipe_wr_cache, mutex)) { 262 pipeclose(NULL, rpipe); 263 pipeclose(NULL, wpipe); 264 return (ENFILE); 265 } 266 267 error = fd_allocfile(&rf, &fd); 268 if (error) 269 goto free2; 270 retval[0] = fd; 271 rf->f_flag = FREAD; 272 rf->f_type = DTYPE_PIPE; 273 rf->f_data = (void *)rpipe; 274 rf->f_ops = &pipeops; 275 276 error = fd_allocfile(&wf, &fd); 277 if (error) 278 goto free3; 279 retval[1] = fd; 280 wf->f_flag = FWRITE; 281 wf->f_type = DTYPE_PIPE; 282 wf->f_data = (void *)wpipe; 283 wf->f_ops = &pipeops; 284 285 rpipe->pipe_peer = wpipe; 286 wpipe->pipe_peer = rpipe; 287 288 fd_affix(p, rf, (int)retval[0]); 289 fd_affix(p, wf, (int)retval[1]); 290 return (0); 291 free3: 292 fd_abort(p, rf, (int)retval[0]); 293 free2: 294 pipeclose(NULL, wpipe); 295 pipeclose(NULL, rpipe); 296 297 return (error); 298 } 299 300 /* 301 * Allocate kva for pipe circular buffer, the space is pageable 302 * This routine will 'realloc' the size of a pipe safely, if it fails 303 * it will retain the old buffer. 304 * If it fails it will return ENOMEM. 305 */ 306 static int 307 pipespace(struct pipe *pipe, int size) 308 { 309 void *buffer; 310 311 /* 312 * Allocate pageable virtual address space. Physical memory is 313 * allocated on demand. 314 */ 315 if (size == PIPE_SIZE && pipe->pipe_kmem != 0) { 316 buffer = (void *)pipe->pipe_kmem; 317 } else { 318 buffer = (void *)uvm_km_alloc(kernel_map, round_page(size), 319 0, UVM_KMF_PAGEABLE); 320 if (buffer == NULL) 321 return (ENOMEM); 322 atomic_add_int(&amountpipekva, size); 323 } 324 325 /* free old resources if we're resizing */ 326 pipe_free_kmem(pipe); 327 pipe->pipe_buffer.buffer = buffer; 328 pipe->pipe_buffer.size = size; 329 pipe->pipe_buffer.in = 0; 330 pipe->pipe_buffer.out = 0; 331 pipe->pipe_buffer.cnt = 0; 332 return (0); 333 } 334 335 /* 336 * Initialize and allocate VM and memory for pipe. 337 */ 338 static int 339 pipe_create(struct pipe **pipep, pool_cache_t cache, kmutex_t *mutex) 340 { 341 struct pipe *pipe; 342 int error; 343 344 pipe = pool_cache_get(cache, PR_WAITOK); 345 KASSERT(pipe != NULL); 346 *pipep = pipe; 347 error = 0; 348 getnanotime(&pipe->pipe_btime); 349 pipe->pipe_atime = pipe->pipe_mtime = pipe->pipe_btime; 350 pipe->pipe_lock = mutex; 351 if (cache == pipe_rd_cache) { 352 error = pipespace(pipe, PIPE_SIZE); 353 } else { 354 pipe->pipe_buffer.buffer = NULL; 355 pipe->pipe_buffer.size = 0; 356 pipe->pipe_buffer.in = 0; 357 pipe->pipe_buffer.out = 0; 358 pipe->pipe_buffer.cnt = 0; 359 } 360 return error; 361 } 362 363 /* 364 * Lock a pipe for I/O, blocking other access 365 * Called with pipe spin lock held. 366 * Return with pipe spin lock released on success. 367 */ 368 static int 369 pipelock(struct pipe *pipe, int catch) 370 { 371 int error; 372 373 KASSERT(mutex_owned(pipe->pipe_lock)); 374 375 while (pipe->pipe_state & PIPE_LOCKFL) { 376 pipe->pipe_state |= PIPE_LWANT; 377 if (catch) { 378 error = cv_wait_sig(&pipe->pipe_lkcv, pipe->pipe_lock); 379 if (error != 0) 380 return error; 381 } else 382 cv_wait(&pipe->pipe_lkcv, pipe->pipe_lock); 383 } 384 385 pipe->pipe_state |= PIPE_LOCKFL; 386 387 return 0; 388 } 389 390 /* 391 * unlock a pipe I/O lock 392 */ 393 static inline void 394 pipeunlock(struct pipe *pipe) 395 { 396 397 KASSERT(pipe->pipe_state & PIPE_LOCKFL); 398 399 pipe->pipe_state &= ~PIPE_LOCKFL; 400 if (pipe->pipe_state & PIPE_LWANT) { 401 pipe->pipe_state &= ~PIPE_LWANT; 402 cv_broadcast(&pipe->pipe_lkcv); 403 } 404 } 405 406 /* 407 * Select/poll wakup. This also sends SIGIO to peer connected to 408 * 'sigpipe' side of pipe. 409 */ 410 static void 411 pipeselwakeup(struct pipe *selp, struct pipe *sigp, int code) 412 { 413 int band; 414 415 switch (code) { 416 case POLL_IN: 417 band = POLLIN|POLLRDNORM; 418 break; 419 case POLL_OUT: 420 band = POLLOUT|POLLWRNORM; 421 break; 422 case POLL_HUP: 423 band = POLLHUP; 424 break; 425 case POLL_ERR: 426 band = POLLERR; 427 break; 428 default: 429 band = 0; 430 #ifdef DIAGNOSTIC 431 printf("bad siginfo code %d in pipe notification.\n", code); 432 #endif 433 break; 434 } 435 436 selnotify(&selp->pipe_sel, band, NOTE_SUBMIT); 437 438 if (sigp == NULL || (sigp->pipe_state & PIPE_ASYNC) == 0) 439 return; 440 441 fownsignal(sigp->pipe_pgid, SIGIO, code, band, selp); 442 } 443 444 static int 445 pipe_read(file_t *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 446 int flags) 447 { 448 struct pipe *rpipe = (struct pipe *) fp->f_data; 449 struct pipebuf *bp = &rpipe->pipe_buffer; 450 kmutex_t *lock = rpipe->pipe_lock; 451 int error; 452 size_t nread = 0; 453 size_t size; 454 size_t ocnt; 455 456 mutex_enter(lock); 457 ++rpipe->pipe_busy; 458 ocnt = bp->cnt; 459 460 again: 461 error = pipelock(rpipe, 1); 462 if (error) 463 goto unlocked_error; 464 465 while (uio->uio_resid) { 466 /* 467 * Normal pipe buffer receive. 468 */ 469 if (bp->cnt > 0) { 470 size = bp->size - bp->out; 471 if (size > bp->cnt) 472 size = bp->cnt; 473 if (size > uio->uio_resid) 474 size = uio->uio_resid; 475 476 mutex_exit(lock); 477 error = uiomove((char *)bp->buffer + bp->out, size, uio); 478 mutex_enter(lock); 479 if (error) 480 break; 481 482 bp->out += size; 483 if (bp->out >= bp->size) 484 bp->out = 0; 485 486 bp->cnt -= size; 487 488 /* 489 * If there is no more to read in the pipe, reset 490 * its pointers to the beginning. This improves 491 * cache hit stats. 492 */ 493 if (bp->cnt == 0) { 494 bp->in = 0; 495 bp->out = 0; 496 } 497 nread += size; 498 continue; 499 } 500 501 #ifndef PIPE_NODIRECT 502 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) { 503 /* 504 * Direct copy, bypassing a kernel buffer. 505 */ 506 void *va; 507 u_int gen; 508 509 KASSERT(rpipe->pipe_state & PIPE_DIRECTW); 510 511 size = rpipe->pipe_map.cnt; 512 if (size > uio->uio_resid) 513 size = uio->uio_resid; 514 515 va = (char *)rpipe->pipe_map.kva + rpipe->pipe_map.pos; 516 gen = rpipe->pipe_map.egen; 517 mutex_exit(lock); 518 519 /* 520 * Consume emap and read the data from loaned pages. 521 */ 522 uvm_emap_consume(gen); 523 error = uiomove(va, size, uio); 524 525 mutex_enter(lock); 526 if (error) 527 break; 528 nread += size; 529 rpipe->pipe_map.pos += size; 530 rpipe->pipe_map.cnt -= size; 531 if (rpipe->pipe_map.cnt == 0) { 532 rpipe->pipe_state &= ~PIPE_DIRECTR; 533 cv_broadcast(&rpipe->pipe_wcv); 534 } 535 continue; 536 } 537 #endif 538 /* 539 * Break if some data was read. 540 */ 541 if (nread > 0) 542 break; 543 544 /* 545 * Detect EOF condition. 546 * Read returns 0 on EOF, no need to set error. 547 */ 548 if (rpipe->pipe_state & PIPE_EOF) 549 break; 550 551 /* 552 * Don't block on non-blocking I/O. 553 */ 554 if (fp->f_flag & FNONBLOCK) { 555 error = EAGAIN; 556 break; 557 } 558 559 /* 560 * Unlock the pipe buffer for our remaining processing. 561 * We will either break out with an error or we will 562 * sleep and relock to loop. 563 */ 564 pipeunlock(rpipe); 565 566 /* 567 * Re-check to see if more direct writes are pending. 568 */ 569 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) 570 goto again; 571 572 /* 573 * We want to read more, wake up select/poll. 574 */ 575 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 576 577 /* 578 * If the "write-side" is blocked, wake it up now. 579 */ 580 cv_broadcast(&rpipe->pipe_wcv); 581 582 /* Now wait until the pipe is filled */ 583 error = cv_wait_sig(&rpipe->pipe_rcv, lock); 584 if (error != 0) 585 goto unlocked_error; 586 goto again; 587 } 588 589 if (error == 0) 590 getnanotime(&rpipe->pipe_atime); 591 pipeunlock(rpipe); 592 593 unlocked_error: 594 --rpipe->pipe_busy; 595 if (rpipe->pipe_busy == 0) { 596 cv_broadcast(&rpipe->pipe_draincv); 597 } 598 if (bp->cnt < MINPIPESIZE) { 599 cv_broadcast(&rpipe->pipe_wcv); 600 } 601 602 /* 603 * If anything was read off the buffer, signal to the writer it's 604 * possible to write more data. Also send signal if we are here for the 605 * first time after last write. 606 */ 607 if ((bp->size - bp->cnt) >= PIPE_BUF 608 && (ocnt != bp->cnt || (rpipe->pipe_state & PIPE_SIGNALR))) { 609 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 610 rpipe->pipe_state &= ~PIPE_SIGNALR; 611 } 612 613 mutex_exit(lock); 614 return (error); 615 } 616 617 #ifndef PIPE_NODIRECT 618 /* 619 * Allocate structure for loan transfer. 620 */ 621 static int 622 pipe_loan_alloc(struct pipe *wpipe, int npages) 623 { 624 vsize_t len; 625 626 len = (vsize_t)npages << PAGE_SHIFT; 627 atomic_add_int(&amountpipekva, len); 628 wpipe->pipe_map.kva = uvm_km_alloc(kernel_map, len, 0, 629 UVM_KMF_VAONLY | UVM_KMF_WAITVA); 630 if (wpipe->pipe_map.kva == 0) { 631 atomic_add_int(&amountpipekva, -len); 632 return (ENOMEM); 633 } 634 635 wpipe->pipe_map.npages = npages; 636 wpipe->pipe_map.pgs = kmem_alloc(npages * sizeof(struct vm_page *), 637 KM_SLEEP); 638 return (0); 639 } 640 641 /* 642 * Free resources allocated for loan transfer. 643 */ 644 static void 645 pipe_loan_free(struct pipe *wpipe) 646 { 647 vsize_t len; 648 649 len = (vsize_t)wpipe->pipe_map.npages << PAGE_SHIFT; 650 uvm_emap_remove(wpipe->pipe_map.kva, len); /* XXX */ 651 uvm_km_free(kernel_map, wpipe->pipe_map.kva, len, UVM_KMF_VAONLY); 652 wpipe->pipe_map.kva = 0; 653 atomic_add_int(&amountpipekva, -len); 654 kmem_free(wpipe->pipe_map.pgs, 655 wpipe->pipe_map.npages * sizeof(struct vm_page *)); 656 wpipe->pipe_map.pgs = NULL; 657 } 658 659 /* 660 * NetBSD direct write, using uvm_loan() mechanism. 661 * This implements the pipe buffer write mechanism. Note that only 662 * a direct write OR a normal pipe write can be pending at any given time. 663 * If there are any characters in the pipe buffer, the direct write will 664 * be deferred until the receiving process grabs all of the bytes from 665 * the pipe buffer. Then the direct mapping write is set-up. 666 * 667 * Called with the long-term pipe lock held. 668 */ 669 static int 670 pipe_direct_write(file_t *fp, struct pipe *wpipe, struct uio *uio) 671 { 672 struct vm_page **pgs; 673 vaddr_t bbase, base, bend; 674 vsize_t blen, bcnt; 675 int error, npages; 676 voff_t bpos; 677 kmutex_t *lock = wpipe->pipe_lock; 678 679 KASSERT(mutex_owned(wpipe->pipe_lock)); 680 KASSERT(wpipe->pipe_map.cnt == 0); 681 682 mutex_exit(lock); 683 684 /* 685 * Handle first PIPE_CHUNK_SIZE bytes of buffer. Deal with buffers 686 * not aligned to PAGE_SIZE. 687 */ 688 bbase = (vaddr_t)uio->uio_iov->iov_base; 689 base = trunc_page(bbase); 690 bend = round_page(bbase + uio->uio_iov->iov_len); 691 blen = bend - base; 692 bpos = bbase - base; 693 694 if (blen > PIPE_DIRECT_CHUNK) { 695 blen = PIPE_DIRECT_CHUNK; 696 bend = base + blen; 697 bcnt = PIPE_DIRECT_CHUNK - bpos; 698 } else { 699 bcnt = uio->uio_iov->iov_len; 700 } 701 npages = blen >> PAGE_SHIFT; 702 703 /* 704 * Free the old kva if we need more pages than we have 705 * allocated. 706 */ 707 if (wpipe->pipe_map.kva != 0 && npages > wpipe->pipe_map.npages) 708 pipe_loan_free(wpipe); 709 710 /* Allocate new kva. */ 711 if (wpipe->pipe_map.kva == 0) { 712 error = pipe_loan_alloc(wpipe, npages); 713 if (error) { 714 mutex_enter(lock); 715 return (error); 716 } 717 } 718 719 /* Loan the write buffer memory from writer process */ 720 pgs = wpipe->pipe_map.pgs; 721 error = uvm_loan(&uio->uio_vmspace->vm_map, base, blen, 722 pgs, UVM_LOAN_TOPAGE); 723 if (error) { 724 pipe_loan_free(wpipe); 725 mutex_enter(lock); 726 return (ENOMEM); /* so that caller fallback to ordinary write */ 727 } 728 729 /* Enter the loaned pages to KVA, produce new emap generation number. */ 730 uvm_emap_enter(wpipe->pipe_map.kva, pgs, npages); 731 wpipe->pipe_map.egen = uvm_emap_produce(); 732 733 /* Now we can put the pipe in direct write mode */ 734 wpipe->pipe_map.pos = bpos; 735 wpipe->pipe_map.cnt = bcnt; 736 737 /* 738 * But before we can let someone do a direct read, we 739 * have to wait until the pipe is drained. Release the 740 * pipe lock while we wait. 741 */ 742 mutex_enter(lock); 743 wpipe->pipe_state |= PIPE_DIRECTW; 744 pipeunlock(wpipe); 745 746 while (error == 0 && wpipe->pipe_buffer.cnt > 0) { 747 cv_broadcast(&wpipe->pipe_rcv); 748 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 749 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 750 error = EPIPE; 751 } 752 753 /* Pipe is drained; next read will off the direct buffer */ 754 wpipe->pipe_state |= PIPE_DIRECTR; 755 756 /* Wait until the reader is done */ 757 while (error == 0 && (wpipe->pipe_state & PIPE_DIRECTR)) { 758 cv_broadcast(&wpipe->pipe_rcv); 759 pipeselwakeup(wpipe, wpipe, POLL_IN); 760 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 761 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 762 error = EPIPE; 763 } 764 765 /* Take pipe out of direct write mode */ 766 wpipe->pipe_state &= ~(PIPE_DIRECTW | PIPE_DIRECTR); 767 768 /* Acquire the pipe lock and cleanup */ 769 (void)pipelock(wpipe, 0); 770 mutex_exit(lock); 771 772 if (pgs != NULL) { 773 /* XXX: uvm_emap_remove */ 774 uvm_unloan(pgs, npages, UVM_LOAN_TOPAGE); 775 } 776 if (error || amountpipekva > maxpipekva) 777 pipe_loan_free(wpipe); 778 779 mutex_enter(lock); 780 if (error) { 781 pipeselwakeup(wpipe, wpipe, POLL_ERR); 782 783 /* 784 * If nothing was read from what we offered, return error 785 * straight on. Otherwise update uio resid first. Caller 786 * will deal with the error condition, returning short 787 * write, error, or restarting the write(2) as appropriate. 788 */ 789 if (wpipe->pipe_map.cnt == bcnt) { 790 wpipe->pipe_map.cnt = 0; 791 cv_broadcast(&wpipe->pipe_wcv); 792 return (error); 793 } 794 795 bcnt -= wpipe->pipe_map.cnt; 796 } 797 798 uio->uio_resid -= bcnt; 799 /* uio_offset not updated, not set/used for write(2) */ 800 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + bcnt; 801 uio->uio_iov->iov_len -= bcnt; 802 if (uio->uio_iov->iov_len == 0) { 803 uio->uio_iov++; 804 uio->uio_iovcnt--; 805 } 806 807 wpipe->pipe_map.cnt = 0; 808 return (error); 809 } 810 #endif /* !PIPE_NODIRECT */ 811 812 static int 813 pipe_write(file_t *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 814 int flags) 815 { 816 struct pipe *wpipe, *rpipe; 817 struct pipebuf *bp; 818 kmutex_t *lock; 819 int error; 820 821 /* We want to write to our peer */ 822 rpipe = (struct pipe *) fp->f_data; 823 lock = rpipe->pipe_lock; 824 error = 0; 825 826 mutex_enter(lock); 827 wpipe = rpipe->pipe_peer; 828 829 /* 830 * Detect loss of pipe read side, issue SIGPIPE if lost. 831 */ 832 if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF) != 0) { 833 mutex_exit(lock); 834 return EPIPE; 835 } 836 ++wpipe->pipe_busy; 837 838 /* Aquire the long-term pipe lock */ 839 if ((error = pipelock(wpipe, 1)) != 0) { 840 --wpipe->pipe_busy; 841 if (wpipe->pipe_busy == 0) { 842 cv_broadcast(&wpipe->pipe_draincv); 843 } 844 mutex_exit(lock); 845 return (error); 846 } 847 848 bp = &wpipe->pipe_buffer; 849 850 /* 851 * If it is advantageous to resize the pipe buffer, do so. 852 */ 853 if ((uio->uio_resid > PIPE_SIZE) && 854 (nbigpipe < maxbigpipes) && 855 #ifndef PIPE_NODIRECT 856 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 857 #endif 858 (bp->size <= PIPE_SIZE) && (bp->cnt == 0)) { 859 860 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) 861 atomic_inc_uint(&nbigpipe); 862 } 863 864 while (uio->uio_resid) { 865 size_t space; 866 867 #ifndef PIPE_NODIRECT 868 /* 869 * Pipe buffered writes cannot be coincidental with 870 * direct writes. Also, only one direct write can be 871 * in progress at any one time. We wait until the currently 872 * executing direct write is completed before continuing. 873 * 874 * We break out if a signal occurs or the reader goes away. 875 */ 876 while (error == 0 && wpipe->pipe_state & PIPE_DIRECTW) { 877 cv_broadcast(&wpipe->pipe_rcv); 878 pipeunlock(wpipe); 879 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 880 (void)pipelock(wpipe, 0); 881 if (wpipe->pipe_state & PIPE_EOF) 882 error = EPIPE; 883 } 884 if (error) 885 break; 886 887 /* 888 * If the transfer is large, we can gain performance if 889 * we do process-to-process copies directly. 890 * If the write is non-blocking, we don't use the 891 * direct write mechanism. 892 * 893 * The direct write mechanism will detect the reader going 894 * away on us. 895 */ 896 if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) && 897 (fp->f_flag & FNONBLOCK) == 0 && 898 (wpipe->pipe_map.kva || (amountpipekva < limitpipekva))) { 899 error = pipe_direct_write(fp, wpipe, uio); 900 901 /* 902 * Break out if error occurred, unless it's ENOMEM. 903 * ENOMEM means we failed to allocate some resources 904 * for direct write, so we just fallback to ordinary 905 * write. If the direct write was successful, 906 * process rest of data via ordinary write. 907 */ 908 if (error == 0) 909 continue; 910 911 if (error != ENOMEM) 912 break; 913 } 914 #endif /* PIPE_NODIRECT */ 915 916 space = bp->size - bp->cnt; 917 918 /* Writes of size <= PIPE_BUF must be atomic. */ 919 if ((space < uio->uio_resid) && (uio->uio_resid <= PIPE_BUF)) 920 space = 0; 921 922 if (space > 0) { 923 int size; /* Transfer size */ 924 int segsize; /* first segment to transfer */ 925 926 /* 927 * Transfer size is minimum of uio transfer 928 * and free space in pipe buffer. 929 */ 930 if (space > uio->uio_resid) 931 size = uio->uio_resid; 932 else 933 size = space; 934 /* 935 * First segment to transfer is minimum of 936 * transfer size and contiguous space in 937 * pipe buffer. If first segment to transfer 938 * is less than the transfer size, we've got 939 * a wraparound in the buffer. 940 */ 941 segsize = bp->size - bp->in; 942 if (segsize > size) 943 segsize = size; 944 945 /* Transfer first segment */ 946 mutex_exit(lock); 947 error = uiomove((char *)bp->buffer + bp->in, segsize, 948 uio); 949 950 if (error == 0 && segsize < size) { 951 /* 952 * Transfer remaining part now, to 953 * support atomic writes. Wraparound 954 * happened. 955 */ 956 KASSERT(bp->in + segsize == bp->size); 957 error = uiomove(bp->buffer, 958 size - segsize, uio); 959 } 960 mutex_enter(lock); 961 if (error) 962 break; 963 964 bp->in += size; 965 if (bp->in >= bp->size) { 966 KASSERT(bp->in == size - segsize + bp->size); 967 bp->in = size - segsize; 968 } 969 970 bp->cnt += size; 971 KASSERT(bp->cnt <= bp->size); 972 } else { 973 /* 974 * If the "read-side" has been blocked, wake it up now. 975 */ 976 cv_broadcast(&wpipe->pipe_rcv); 977 978 /* 979 * Don't block on non-blocking I/O. 980 */ 981 if (fp->f_flag & FNONBLOCK) { 982 error = EAGAIN; 983 break; 984 } 985 986 /* 987 * We have no more space and have something to offer, 988 * wake up select/poll. 989 */ 990 if (bp->cnt) 991 pipeselwakeup(wpipe, wpipe, POLL_IN); 992 993 pipeunlock(wpipe); 994 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 995 (void)pipelock(wpipe, 0); 996 if (error != 0) 997 break; 998 /* 999 * If read side wants to go away, we just issue a signal 1000 * to ourselves. 1001 */ 1002 if (wpipe->pipe_state & PIPE_EOF) { 1003 error = EPIPE; 1004 break; 1005 } 1006 } 1007 } 1008 1009 --wpipe->pipe_busy; 1010 if (wpipe->pipe_busy == 0) { 1011 cv_broadcast(&wpipe->pipe_draincv); 1012 } 1013 if (bp->cnt > 0) { 1014 cv_broadcast(&wpipe->pipe_rcv); 1015 } 1016 1017 /* 1018 * Don't return EPIPE if I/O was successful 1019 */ 1020 if (error == EPIPE && bp->cnt == 0 && uio->uio_resid == 0) 1021 error = 0; 1022 1023 if (error == 0) 1024 getnanotime(&wpipe->pipe_mtime); 1025 1026 /* 1027 * We have something to offer, wake up select/poll. 1028 * wpipe->pipe_map.cnt is always 0 in this point (direct write 1029 * is only done synchronously), so check only wpipe->pipe_buffer.cnt 1030 */ 1031 if (bp->cnt) 1032 pipeselwakeup(wpipe, wpipe, POLL_IN); 1033 1034 /* 1035 * Arrange for next read(2) to do a signal. 1036 */ 1037 wpipe->pipe_state |= PIPE_SIGNALR; 1038 1039 pipeunlock(wpipe); 1040 mutex_exit(lock); 1041 return (error); 1042 } 1043 1044 /* 1045 * We implement a very minimal set of ioctls for compatibility with sockets. 1046 */ 1047 int 1048 pipe_ioctl(file_t *fp, u_long cmd, void *data) 1049 { 1050 struct pipe *pipe = fp->f_data; 1051 kmutex_t *lock = pipe->pipe_lock; 1052 1053 switch (cmd) { 1054 1055 case FIONBIO: 1056 return (0); 1057 1058 case FIOASYNC: 1059 mutex_enter(lock); 1060 if (*(int *)data) { 1061 pipe->pipe_state |= PIPE_ASYNC; 1062 } else { 1063 pipe->pipe_state &= ~PIPE_ASYNC; 1064 } 1065 mutex_exit(lock); 1066 return (0); 1067 1068 case FIONREAD: 1069 mutex_enter(lock); 1070 #ifndef PIPE_NODIRECT 1071 if (pipe->pipe_state & PIPE_DIRECTW) 1072 *(int *)data = pipe->pipe_map.cnt; 1073 else 1074 #endif 1075 *(int *)data = pipe->pipe_buffer.cnt; 1076 mutex_exit(lock); 1077 return (0); 1078 1079 case FIONWRITE: 1080 /* Look at other side */ 1081 pipe = pipe->pipe_peer; 1082 mutex_enter(lock); 1083 #ifndef PIPE_NODIRECT 1084 if (pipe->pipe_state & PIPE_DIRECTW) 1085 *(int *)data = pipe->pipe_map.cnt; 1086 else 1087 #endif 1088 *(int *)data = pipe->pipe_buffer.cnt; 1089 mutex_exit(lock); 1090 return (0); 1091 1092 case FIONSPACE: 1093 /* Look at other side */ 1094 pipe = pipe->pipe_peer; 1095 mutex_enter(lock); 1096 #ifndef PIPE_NODIRECT 1097 /* 1098 * If we're in direct-mode, we don't really have a 1099 * send queue, and any other write will block. Thus 1100 * zero seems like the best answer. 1101 */ 1102 if (pipe->pipe_state & PIPE_DIRECTW) 1103 *(int *)data = 0; 1104 else 1105 #endif 1106 *(int *)data = pipe->pipe_buffer.size - 1107 pipe->pipe_buffer.cnt; 1108 mutex_exit(lock); 1109 return (0); 1110 1111 case TIOCSPGRP: 1112 case FIOSETOWN: 1113 return fsetown(&pipe->pipe_pgid, cmd, data); 1114 1115 case TIOCGPGRP: 1116 case FIOGETOWN: 1117 return fgetown(pipe->pipe_pgid, cmd, data); 1118 1119 } 1120 return (EPASSTHROUGH); 1121 } 1122 1123 int 1124 pipe_poll(file_t *fp, int events) 1125 { 1126 struct pipe *rpipe = fp->f_data; 1127 struct pipe *wpipe; 1128 int eof = 0; 1129 int revents = 0; 1130 1131 mutex_enter(rpipe->pipe_lock); 1132 wpipe = rpipe->pipe_peer; 1133 1134 if (events & (POLLIN | POLLRDNORM)) 1135 if ((rpipe->pipe_buffer.cnt > 0) || 1136 #ifndef PIPE_NODIRECT 1137 (rpipe->pipe_state & PIPE_DIRECTR) || 1138 #endif 1139 (rpipe->pipe_state & PIPE_EOF)) 1140 revents |= events & (POLLIN | POLLRDNORM); 1141 1142 eof |= (rpipe->pipe_state & PIPE_EOF); 1143 1144 if (wpipe == NULL) 1145 revents |= events & (POLLOUT | POLLWRNORM); 1146 else { 1147 if (events & (POLLOUT | POLLWRNORM)) 1148 if ((wpipe->pipe_state & PIPE_EOF) || ( 1149 #ifndef PIPE_NODIRECT 1150 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 1151 #endif 1152 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 1153 revents |= events & (POLLOUT | POLLWRNORM); 1154 1155 eof |= (wpipe->pipe_state & PIPE_EOF); 1156 } 1157 1158 if (wpipe == NULL || eof) 1159 revents |= POLLHUP; 1160 1161 if (revents == 0) { 1162 if (events & (POLLIN | POLLRDNORM)) 1163 selrecord(curlwp, &rpipe->pipe_sel); 1164 1165 if (events & (POLLOUT | POLLWRNORM)) 1166 selrecord(curlwp, &wpipe->pipe_sel); 1167 } 1168 mutex_exit(rpipe->pipe_lock); 1169 1170 return (revents); 1171 } 1172 1173 static int 1174 pipe_stat(file_t *fp, struct stat *ub) 1175 { 1176 struct pipe *pipe = fp->f_data; 1177 1178 mutex_enter(pipe->pipe_lock); 1179 memset(ub, 0, sizeof(*ub)); 1180 ub->st_mode = S_IFIFO | S_IRUSR | S_IWUSR; 1181 ub->st_blksize = pipe->pipe_buffer.size; 1182 if (ub->st_blksize == 0 && pipe->pipe_peer) 1183 ub->st_blksize = pipe->pipe_peer->pipe_buffer.size; 1184 ub->st_size = pipe->pipe_buffer.cnt; 1185 ub->st_blocks = (ub->st_size) ? 1 : 0; 1186 ub->st_atimespec = pipe->pipe_atime; 1187 ub->st_mtimespec = pipe->pipe_mtime; 1188 ub->st_ctimespec = ub->st_birthtimespec = pipe->pipe_btime; 1189 ub->st_uid = kauth_cred_geteuid(fp->f_cred); 1190 ub->st_gid = kauth_cred_getegid(fp->f_cred); 1191 1192 /* 1193 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 1194 * XXX (st_dev, st_ino) should be unique. 1195 */ 1196 mutex_exit(pipe->pipe_lock); 1197 return 0; 1198 } 1199 1200 static int 1201 pipe_close(file_t *fp) 1202 { 1203 struct pipe *pipe = fp->f_data; 1204 1205 fp->f_data = NULL; 1206 pipeclose(fp, pipe); 1207 return (0); 1208 } 1209 1210 static void 1211 pipe_free_kmem(struct pipe *pipe) 1212 { 1213 1214 if (pipe->pipe_buffer.buffer != NULL) { 1215 if (pipe->pipe_buffer.size > PIPE_SIZE) { 1216 atomic_dec_uint(&nbigpipe); 1217 } 1218 if (pipe->pipe_buffer.buffer != (void *)pipe->pipe_kmem) { 1219 uvm_km_free(kernel_map, 1220 (vaddr_t)pipe->pipe_buffer.buffer, 1221 pipe->pipe_buffer.size, UVM_KMF_PAGEABLE); 1222 atomic_add_int(&amountpipekva, 1223 -pipe->pipe_buffer.size); 1224 } 1225 pipe->pipe_buffer.buffer = NULL; 1226 } 1227 #ifndef PIPE_NODIRECT 1228 if (pipe->pipe_map.kva != 0) { 1229 pipe_loan_free(pipe); 1230 pipe->pipe_map.cnt = 0; 1231 pipe->pipe_map.kva = 0; 1232 pipe->pipe_map.pos = 0; 1233 pipe->pipe_map.npages = 0; 1234 } 1235 #endif /* !PIPE_NODIRECT */ 1236 } 1237 1238 /* 1239 * Shutdown the pipe. 1240 */ 1241 static void 1242 pipeclose(file_t *fp, struct pipe *pipe) 1243 { 1244 kmutex_t *lock; 1245 struct pipe *ppipe; 1246 1247 if (pipe == NULL) 1248 return; 1249 1250 KASSERT(cv_is_valid(&pipe->pipe_rcv)); 1251 KASSERT(cv_is_valid(&pipe->pipe_wcv)); 1252 KASSERT(cv_is_valid(&pipe->pipe_draincv)); 1253 KASSERT(cv_is_valid(&pipe->pipe_lkcv)); 1254 1255 lock = pipe->pipe_lock; 1256 mutex_enter(lock); 1257 pipeselwakeup(pipe, pipe, POLL_HUP); 1258 1259 /* 1260 * If the other side is blocked, wake it up saying that 1261 * we want to close it down. 1262 */ 1263 pipe->pipe_state |= PIPE_EOF; 1264 if (pipe->pipe_busy) { 1265 while (pipe->pipe_busy) { 1266 cv_broadcast(&pipe->pipe_wcv); 1267 cv_wait_sig(&pipe->pipe_draincv, lock); 1268 } 1269 } 1270 1271 /* 1272 * Disconnect from peer. 1273 */ 1274 if ((ppipe = pipe->pipe_peer) != NULL) { 1275 pipeselwakeup(ppipe, ppipe, POLL_HUP); 1276 ppipe->pipe_state |= PIPE_EOF; 1277 cv_broadcast(&ppipe->pipe_rcv); 1278 ppipe->pipe_peer = NULL; 1279 } 1280 1281 /* 1282 * Any knote objects still left in the list are 1283 * the one attached by peer. Since no one will 1284 * traverse this list, we just clear it. 1285 */ 1286 SLIST_INIT(&pipe->pipe_sel.sel_klist); 1287 1288 KASSERT((pipe->pipe_state & PIPE_LOCKFL) == 0); 1289 mutex_exit(lock); 1290 1291 /* 1292 * Free resources. 1293 */ 1294 pipe->pipe_pgid = 0; 1295 pipe->pipe_state = PIPE_SIGNALR; 1296 pipe_free_kmem(pipe); 1297 if (pipe->pipe_kmem != 0) { 1298 pool_cache_put(pipe_rd_cache, pipe); 1299 } else { 1300 pool_cache_put(pipe_wr_cache, pipe); 1301 } 1302 mutex_obj_free(lock); 1303 } 1304 1305 static void 1306 filt_pipedetach(struct knote *kn) 1307 { 1308 struct pipe *pipe; 1309 kmutex_t *lock; 1310 1311 pipe = ((file_t *)kn->kn_obj)->f_data; 1312 lock = pipe->pipe_lock; 1313 1314 mutex_enter(lock); 1315 1316 switch(kn->kn_filter) { 1317 case EVFILT_WRITE: 1318 /* Need the peer structure, not our own. */ 1319 pipe = pipe->pipe_peer; 1320 1321 /* If reader end already closed, just return. */ 1322 if (pipe == NULL) { 1323 mutex_exit(lock); 1324 return; 1325 } 1326 1327 break; 1328 default: 1329 /* Nothing to do. */ 1330 break; 1331 } 1332 1333 KASSERT(kn->kn_hook == pipe); 1334 SLIST_REMOVE(&pipe->pipe_sel.sel_klist, kn, knote, kn_selnext); 1335 mutex_exit(lock); 1336 } 1337 1338 static int 1339 filt_piperead(struct knote *kn, long hint) 1340 { 1341 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_data; 1342 struct pipe *wpipe; 1343 1344 if ((hint & NOTE_SUBMIT) == 0) { 1345 mutex_enter(rpipe->pipe_lock); 1346 } 1347 wpipe = rpipe->pipe_peer; 1348 kn->kn_data = rpipe->pipe_buffer.cnt; 1349 1350 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1351 kn->kn_data = rpipe->pipe_map.cnt; 1352 1353 if ((rpipe->pipe_state & PIPE_EOF) || 1354 (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1355 kn->kn_flags |= EV_EOF; 1356 if ((hint & NOTE_SUBMIT) == 0) { 1357 mutex_exit(rpipe->pipe_lock); 1358 } 1359 return (1); 1360 } 1361 1362 if ((hint & NOTE_SUBMIT) == 0) { 1363 mutex_exit(rpipe->pipe_lock); 1364 } 1365 return (kn->kn_data > 0); 1366 } 1367 1368 static int 1369 filt_pipewrite(struct knote *kn, long hint) 1370 { 1371 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_data; 1372 struct pipe *wpipe; 1373 1374 if ((hint & NOTE_SUBMIT) == 0) { 1375 mutex_enter(rpipe->pipe_lock); 1376 } 1377 wpipe = rpipe->pipe_peer; 1378 1379 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1380 kn->kn_data = 0; 1381 kn->kn_flags |= EV_EOF; 1382 if ((hint & NOTE_SUBMIT) == 0) { 1383 mutex_exit(rpipe->pipe_lock); 1384 } 1385 return (1); 1386 } 1387 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1388 if (wpipe->pipe_state & PIPE_DIRECTW) 1389 kn->kn_data = 0; 1390 1391 if ((hint & NOTE_SUBMIT) == 0) { 1392 mutex_exit(rpipe->pipe_lock); 1393 } 1394 return (kn->kn_data >= PIPE_BUF); 1395 } 1396 1397 static const struct filterops pipe_rfiltops = 1398 { 1, NULL, filt_pipedetach, filt_piperead }; 1399 static const struct filterops pipe_wfiltops = 1400 { 1, NULL, filt_pipedetach, filt_pipewrite }; 1401 1402 static int 1403 pipe_kqfilter(file_t *fp, struct knote *kn) 1404 { 1405 struct pipe *pipe; 1406 kmutex_t *lock; 1407 1408 pipe = ((file_t *)kn->kn_obj)->f_data; 1409 lock = pipe->pipe_lock; 1410 1411 mutex_enter(lock); 1412 1413 switch (kn->kn_filter) { 1414 case EVFILT_READ: 1415 kn->kn_fop = &pipe_rfiltops; 1416 break; 1417 case EVFILT_WRITE: 1418 kn->kn_fop = &pipe_wfiltops; 1419 pipe = pipe->pipe_peer; 1420 if (pipe == NULL) { 1421 /* Other end of pipe has been closed. */ 1422 mutex_exit(lock); 1423 return (EBADF); 1424 } 1425 break; 1426 default: 1427 mutex_exit(lock); 1428 return (EINVAL); 1429 } 1430 1431 kn->kn_hook = pipe; 1432 SLIST_INSERT_HEAD(&pipe->pipe_sel.sel_klist, kn, kn_selnext); 1433 mutex_exit(lock); 1434 1435 return (0); 1436 } 1437 1438 /* 1439 * Handle pipe sysctls. 1440 */ 1441 SYSCTL_SETUP(sysctl_kern_pipe_setup, "sysctl kern.pipe subtree setup") 1442 { 1443 1444 sysctl_createv(clog, 0, NULL, NULL, 1445 CTLFLAG_PERMANENT, 1446 CTLTYPE_NODE, "kern", NULL, 1447 NULL, 0, NULL, 0, 1448 CTL_KERN, CTL_EOL); 1449 sysctl_createv(clog, 0, NULL, NULL, 1450 CTLFLAG_PERMANENT, 1451 CTLTYPE_NODE, "pipe", 1452 SYSCTL_DESCR("Pipe settings"), 1453 NULL, 0, NULL, 0, 1454 CTL_KERN, KERN_PIPE, CTL_EOL); 1455 1456 sysctl_createv(clog, 0, NULL, NULL, 1457 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1458 CTLTYPE_INT, "maxkvasz", 1459 SYSCTL_DESCR("Maximum amount of kernel memory to be " 1460 "used for pipes"), 1461 NULL, 0, &maxpipekva, 0, 1462 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXKVASZ, CTL_EOL); 1463 sysctl_createv(clog, 0, NULL, NULL, 1464 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1465 CTLTYPE_INT, "maxloankvasz", 1466 SYSCTL_DESCR("Limit for direct transfers via page loan"), 1467 NULL, 0, &limitpipekva, 0, 1468 CTL_KERN, KERN_PIPE, KERN_PIPE_LIMITKVA, CTL_EOL); 1469 sysctl_createv(clog, 0, NULL, NULL, 1470 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1471 CTLTYPE_INT, "maxbigpipes", 1472 SYSCTL_DESCR("Maximum number of \"big\" pipes"), 1473 NULL, 0, &maxbigpipes, 0, 1474 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXBIGPIPES, CTL_EOL); 1475 sysctl_createv(clog, 0, NULL, NULL, 1476 CTLFLAG_PERMANENT, 1477 CTLTYPE_INT, "nbigpipes", 1478 SYSCTL_DESCR("Number of \"big\" pipes"), 1479 NULL, 0, &nbigpipe, 0, 1480 CTL_KERN, KERN_PIPE, KERN_PIPE_NBIGPIPES, CTL_EOL); 1481 sysctl_createv(clog, 0, NULL, NULL, 1482 CTLFLAG_PERMANENT, 1483 CTLTYPE_INT, "kvasize", 1484 SYSCTL_DESCR("Amount of kernel memory consumed by pipe " 1485 "buffers"), 1486 NULL, 0, &amountpipekva, 0, 1487 CTL_KERN, KERN_PIPE, KERN_PIPE_KVASIZE, CTL_EOL); 1488 } 1489