1 /* $NetBSD: sys_pipe.c,v 1.103 2008/09/17 14:00:41 pooka Exp $ */ 2 3 /*- 4 * Copyright (c) 2003, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Paul Kranenburg, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1996 John S. Dyson 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice immediately at the beginning of the file, without modification, 41 * this list of conditions, and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. Absolutely no warranty of function or purpose is made by the author 46 * John S. Dyson. 47 * 4. Modifications may be freely made to this file if the above conditions 48 * are met. 49 * 50 * $FreeBSD: src/sys/kern/sys_pipe.c,v 1.95 2002/03/09 22:06:31 alfred Exp $ 51 */ 52 53 /* 54 * This file contains a high-performance replacement for the socket-based 55 * pipes scheme originally used in FreeBSD/4.4Lite. It does not support 56 * all features of sockets, but does do everything that pipes normally 57 * do. 58 * 59 * Adaption for NetBSD UVM, including uvm_loan() based direct write, was 60 * written by Jaromir Dolecek. 61 */ 62 63 /* 64 * This code has two modes of operation, a small write mode and a large 65 * write mode. The small write mode acts like conventional pipes with 66 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 67 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 68 * and PIPE_SIZE in size it is mapped read-only into the kernel address space 69 * using the UVM page loan facility from where the receiving process can copy 70 * the data directly from the pages in the sending process. 71 * 72 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 73 * happen for small transfers so that the system will not spend all of 74 * its time context switching. PIPE_SIZE is constrained by the 75 * amount of kernel virtual memory. 76 */ 77 78 #include <sys/cdefs.h> 79 __KERNEL_RCSID(0, "$NetBSD: sys_pipe.c,v 1.103 2008/09/17 14:00:41 pooka Exp $"); 80 81 #include <sys/param.h> 82 #include <sys/systm.h> 83 #include <sys/proc.h> 84 #include <sys/fcntl.h> 85 #include <sys/file.h> 86 #include <sys/filedesc.h> 87 #include <sys/filio.h> 88 #include <sys/kernel.h> 89 #include <sys/ttycom.h> 90 #include <sys/stat.h> 91 #include <sys/poll.h> 92 #include <sys/signalvar.h> 93 #include <sys/vnode.h> 94 #include <sys/uio.h> 95 #include <sys/select.h> 96 #include <sys/mount.h> 97 #include <sys/syscallargs.h> 98 #include <sys/sysctl.h> 99 #include <sys/kauth.h> 100 #include <sys/atomic.h> 101 #include <sys/pipe.h> 102 103 #include <uvm/uvm.h> 104 105 /* 106 * Use this define if you want to disable *fancy* VM things. Expect an 107 * approx 30% decrease in transfer rate. 108 */ 109 /* #define PIPE_NODIRECT */ 110 111 /* 112 * interfaces to the outside world 113 */ 114 static int pipe_read(struct file *fp, off_t *offset, struct uio *uio, 115 kauth_cred_t cred, int flags); 116 static int pipe_write(struct file *fp, off_t *offset, struct uio *uio, 117 kauth_cred_t cred, int flags); 118 static int pipe_close(struct file *fp); 119 static int pipe_poll(struct file *fp, int events); 120 static int pipe_kqfilter(struct file *fp, struct knote *kn); 121 static int pipe_stat(struct file *fp, struct stat *sb); 122 static int pipe_ioctl(struct file *fp, u_long cmd, void *data); 123 124 static const struct fileops pipeops = { 125 pipe_read, pipe_write, pipe_ioctl, fnullop_fcntl, pipe_poll, 126 pipe_stat, pipe_close, pipe_kqfilter 127 }; 128 129 /* 130 * Single mutex shared between both ends of the pipe. 131 */ 132 133 struct pipe_mutex { 134 kmutex_t pm_mutex; 135 u_int pm_refcnt; 136 }; 137 138 /* 139 * Default pipe buffer size(s), this can be kind-of large now because pipe 140 * space is pageable. The pipe code will try to maintain locality of 141 * reference for performance reasons, so small amounts of outstanding I/O 142 * will not wipe the cache. 143 */ 144 #define MINPIPESIZE (PIPE_SIZE/3) 145 #define MAXPIPESIZE (2*PIPE_SIZE/3) 146 147 /* 148 * Maximum amount of kva for pipes -- this is kind-of a soft limit, but 149 * is there so that on large systems, we don't exhaust it. 150 */ 151 #define MAXPIPEKVA (8*1024*1024) 152 static u_int maxpipekva = MAXPIPEKVA; 153 154 /* 155 * Limit for direct transfers, we cannot, of course limit 156 * the amount of kva for pipes in general though. 157 */ 158 #define LIMITPIPEKVA (16*1024*1024) 159 static u_int limitpipekva = LIMITPIPEKVA; 160 161 /* 162 * Limit the number of "big" pipes 163 */ 164 #define LIMITBIGPIPES 32 165 static u_int maxbigpipes = LIMITBIGPIPES; 166 static u_int nbigpipe = 0; 167 168 /* 169 * Amount of KVA consumed by pipe buffers. 170 */ 171 static u_int amountpipekva = 0; 172 173 static void pipeclose(struct file *fp, struct pipe *pipe); 174 static void pipe_free_kmem(struct pipe *pipe); 175 static int pipe_create(struct pipe **pipep, int allockva, struct pipe_mutex *); 176 static int pipelock(struct pipe *pipe, int catch); 177 static inline void pipeunlock(struct pipe *pipe); 178 static void pipeselwakeup(struct pipe *pipe, struct pipe *sigp, int code); 179 #ifndef PIPE_NODIRECT 180 static int pipe_direct_write(struct file *fp, struct pipe *wpipe, 181 struct uio *uio); 182 #endif 183 static int pipespace(struct pipe *pipe, int size); 184 185 #ifndef PIPE_NODIRECT 186 static int pipe_loan_alloc(struct pipe *, int); 187 static void pipe_loan_free(struct pipe *); 188 #endif /* PIPE_NODIRECT */ 189 190 static int pipe_mutex_ctor(void *, void *, int); 191 static void pipe_mutex_dtor(void *, void *); 192 193 static pool_cache_t pipe_cache; 194 static pool_cache_t pipe_mutex_cache; 195 196 void 197 pipe_init(void) 198 { 199 200 pipe_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "pipepl", 201 NULL, IPL_NONE, NULL, NULL, NULL); 202 KASSERT(pipe_cache != NULL); 203 204 pipe_mutex_cache = pool_cache_init(sizeof(struct pipe_mutex), 205 coherency_unit, 0, 0, "pipemtxpl", NULL, IPL_NONE, pipe_mutex_ctor, 206 pipe_mutex_dtor, NULL); 207 KASSERT(pipe_cache != NULL); 208 } 209 210 static int 211 pipe_mutex_ctor(void *arg, void *obj, int flag) 212 { 213 struct pipe_mutex *pm = obj; 214 215 mutex_init(&pm->pm_mutex, MUTEX_DEFAULT, IPL_NONE); 216 pm->pm_refcnt = 0; 217 218 return 0; 219 } 220 221 static void 222 pipe_mutex_dtor(void *arg, void *obj) 223 { 224 struct pipe_mutex *pm = obj; 225 226 KASSERT(pm->pm_refcnt == 0); 227 228 mutex_destroy(&pm->pm_mutex); 229 } 230 231 /* 232 * The pipe system call for the DTYPE_PIPE type of pipes 233 */ 234 235 /* ARGSUSED */ 236 int 237 sys_pipe(struct lwp *l, const void *v, register_t *retval) 238 { 239 struct file *rf, *wf; 240 struct pipe *rpipe, *wpipe; 241 struct pipe_mutex *mutex; 242 int fd, error; 243 proc_t *p; 244 245 p = curproc; 246 rpipe = wpipe = NULL; 247 mutex = pool_cache_get(pipe_mutex_cache, PR_WAITOK); 248 if (mutex == NULL) 249 return (ENOMEM); 250 if (pipe_create(&rpipe, 1, mutex) || pipe_create(&wpipe, 0, mutex)) { 251 pipeclose(NULL, rpipe); 252 pipeclose(NULL, wpipe); 253 return (ENFILE); 254 } 255 256 error = fd_allocfile(&rf, &fd); 257 if (error) 258 goto free2; 259 retval[0] = fd; 260 rf->f_flag = FREAD; 261 rf->f_type = DTYPE_PIPE; 262 rf->f_data = (void *)rpipe; 263 rf->f_ops = &pipeops; 264 265 error = fd_allocfile(&wf, &fd); 266 if (error) 267 goto free3; 268 retval[1] = fd; 269 wf->f_flag = FWRITE; 270 wf->f_type = DTYPE_PIPE; 271 wf->f_data = (void *)wpipe; 272 wf->f_ops = &pipeops; 273 274 rpipe->pipe_peer = wpipe; 275 wpipe->pipe_peer = rpipe; 276 277 fd_affix(p, rf, (int)retval[0]); 278 fd_affix(p, wf, (int)retval[1]); 279 return (0); 280 free3: 281 fd_abort(p, rf, (int)retval[0]); 282 free2: 283 pipeclose(NULL, wpipe); 284 pipeclose(NULL, rpipe); 285 286 return (error); 287 } 288 289 /* 290 * Allocate kva for pipe circular buffer, the space is pageable 291 * This routine will 'realloc' the size of a pipe safely, if it fails 292 * it will retain the old buffer. 293 * If it fails it will return ENOMEM. 294 */ 295 static int 296 pipespace(struct pipe *pipe, int size) 297 { 298 void *buffer; 299 /* 300 * Allocate pageable virtual address space. Physical memory is 301 * allocated on demand. 302 */ 303 buffer = (void *) uvm_km_alloc(kernel_map, round_page(size), 0, 304 UVM_KMF_PAGEABLE); 305 if (buffer == NULL) 306 return (ENOMEM); 307 308 /* free old resources if we're resizing */ 309 pipe_free_kmem(pipe); 310 pipe->pipe_buffer.buffer = buffer; 311 pipe->pipe_buffer.size = size; 312 pipe->pipe_buffer.in = 0; 313 pipe->pipe_buffer.out = 0; 314 pipe->pipe_buffer.cnt = 0; 315 atomic_add_int(&amountpipekva, pipe->pipe_buffer.size); 316 return (0); 317 } 318 319 /* 320 * Initialize and allocate VM and memory for pipe. 321 */ 322 static int 323 pipe_create(struct pipe **pipep, int allockva, struct pipe_mutex *mutex) 324 { 325 struct pipe *pipe; 326 int error; 327 328 pipe = *pipep = pool_cache_get(pipe_cache, PR_WAITOK); 329 mutex->pm_refcnt++; 330 331 /* Initialize */ 332 memset(pipe, 0, sizeof(struct pipe)); 333 pipe->pipe_state = PIPE_SIGNALR; 334 335 getmicrotime(&pipe->pipe_ctime); 336 pipe->pipe_atime = pipe->pipe_ctime; 337 pipe->pipe_mtime = pipe->pipe_ctime; 338 pipe->pipe_lock = &mutex->pm_mutex; 339 cv_init(&pipe->pipe_rcv, "piperd"); 340 cv_init(&pipe->pipe_wcv, "pipewr"); 341 cv_init(&pipe->pipe_draincv, "pipedrain"); 342 cv_init(&pipe->pipe_lkcv, "pipelk"); 343 selinit(&pipe->pipe_sel); 344 345 if (allockva && (error = pipespace(pipe, PIPE_SIZE))) 346 return (error); 347 348 return (0); 349 } 350 351 352 /* 353 * Lock a pipe for I/O, blocking other access 354 * Called with pipe spin lock held. 355 * Return with pipe spin lock released on success. 356 */ 357 static int 358 pipelock(struct pipe *pipe, int catch) 359 { 360 int error; 361 362 KASSERT(mutex_owned(pipe->pipe_lock)); 363 364 while (pipe->pipe_state & PIPE_LOCKFL) { 365 pipe->pipe_state |= PIPE_LWANT; 366 if (catch) { 367 error = cv_wait_sig(&pipe->pipe_lkcv, pipe->pipe_lock); 368 if (error != 0) 369 return error; 370 } else 371 cv_wait(&pipe->pipe_lkcv, pipe->pipe_lock); 372 } 373 374 pipe->pipe_state |= PIPE_LOCKFL; 375 376 return 0; 377 } 378 379 /* 380 * unlock a pipe I/O lock 381 */ 382 static inline void 383 pipeunlock(struct pipe *pipe) 384 { 385 386 KASSERT(pipe->pipe_state & PIPE_LOCKFL); 387 388 pipe->pipe_state &= ~PIPE_LOCKFL; 389 if (pipe->pipe_state & PIPE_LWANT) { 390 pipe->pipe_state &= ~PIPE_LWANT; 391 cv_broadcast(&pipe->pipe_lkcv); 392 } 393 } 394 395 /* 396 * Select/poll wakup. This also sends SIGIO to peer connected to 397 * 'sigpipe' side of pipe. 398 */ 399 static void 400 pipeselwakeup(struct pipe *selp, struct pipe *sigp, int code) 401 { 402 int band; 403 404 switch (code) { 405 case POLL_IN: 406 band = POLLIN|POLLRDNORM; 407 break; 408 case POLL_OUT: 409 band = POLLOUT|POLLWRNORM; 410 break; 411 case POLL_HUP: 412 band = POLLHUP; 413 break; 414 #if POLL_HUP != POLL_ERR 415 case POLL_ERR: 416 band = POLLERR; 417 break; 418 #endif 419 default: 420 band = 0; 421 #ifdef DIAGNOSTIC 422 printf("bad siginfo code %d in pipe notification.\n", code); 423 #endif 424 break; 425 } 426 427 selnotify(&selp->pipe_sel, band, NOTE_SUBMIT); 428 429 if (sigp == NULL || (sigp->pipe_state & PIPE_ASYNC) == 0) 430 return; 431 432 fownsignal(sigp->pipe_pgid, SIGIO, code, band, selp); 433 } 434 435 /* ARGSUSED */ 436 static int 437 pipe_read(struct file *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 438 int flags) 439 { 440 struct pipe *rpipe = (struct pipe *) fp->f_data; 441 struct pipebuf *bp = &rpipe->pipe_buffer; 442 kmutex_t *lock = rpipe->pipe_lock; 443 int error; 444 size_t nread = 0; 445 size_t size; 446 size_t ocnt; 447 448 mutex_enter(lock); 449 ++rpipe->pipe_busy; 450 ocnt = bp->cnt; 451 452 again: 453 error = pipelock(rpipe, 1); 454 if (error) 455 goto unlocked_error; 456 457 while (uio->uio_resid) { 458 /* 459 * normal pipe buffer receive 460 */ 461 if (bp->cnt > 0) { 462 size = bp->size - bp->out; 463 if (size > bp->cnt) 464 size = bp->cnt; 465 if (size > uio->uio_resid) 466 size = uio->uio_resid; 467 468 mutex_exit(lock); 469 error = uiomove((char *)bp->buffer + bp->out, size, uio); 470 mutex_enter(lock); 471 if (error) 472 break; 473 474 bp->out += size; 475 if (bp->out >= bp->size) 476 bp->out = 0; 477 478 bp->cnt -= size; 479 480 /* 481 * If there is no more to read in the pipe, reset 482 * its pointers to the beginning. This improves 483 * cache hit stats. 484 */ 485 if (bp->cnt == 0) { 486 bp->in = 0; 487 bp->out = 0; 488 } 489 nread += size; 490 continue; 491 } 492 493 #ifndef PIPE_NODIRECT 494 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) { 495 /* 496 * Direct copy, bypassing a kernel buffer. 497 */ 498 void * va; 499 500 KASSERT(rpipe->pipe_state & PIPE_DIRECTW); 501 502 size = rpipe->pipe_map.cnt; 503 if (size > uio->uio_resid) 504 size = uio->uio_resid; 505 506 va = (char *)rpipe->pipe_map.kva + rpipe->pipe_map.pos; 507 mutex_exit(lock); 508 error = uiomove(va, size, uio); 509 mutex_enter(lock); 510 if (error) 511 break; 512 nread += size; 513 rpipe->pipe_map.pos += size; 514 rpipe->pipe_map.cnt -= size; 515 if (rpipe->pipe_map.cnt == 0) { 516 rpipe->pipe_state &= ~PIPE_DIRECTR; 517 cv_broadcast(&rpipe->pipe_wcv); 518 } 519 continue; 520 } 521 #endif 522 /* 523 * Break if some data was read. 524 */ 525 if (nread > 0) 526 break; 527 528 /* 529 * detect EOF condition 530 * read returns 0 on EOF, no need to set error 531 */ 532 if (rpipe->pipe_state & PIPE_EOF) 533 break; 534 535 /* 536 * don't block on non-blocking I/O 537 */ 538 if (fp->f_flag & FNONBLOCK) { 539 error = EAGAIN; 540 break; 541 } 542 543 /* 544 * Unlock the pipe buffer for our remaining processing. 545 * We will either break out with an error or we will 546 * sleep and relock to loop. 547 */ 548 pipeunlock(rpipe); 549 550 /* 551 * Re-check to see if more direct writes are pending. 552 */ 553 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) 554 goto again; 555 556 /* 557 * We want to read more, wake up select/poll. 558 */ 559 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_IN); 560 561 /* 562 * If the "write-side" is blocked, wake it up now. 563 */ 564 cv_broadcast(&rpipe->pipe_wcv); 565 566 /* Now wait until the pipe is filled */ 567 error = cv_wait_sig(&rpipe->pipe_rcv, lock); 568 if (error != 0) 569 goto unlocked_error; 570 goto again; 571 } 572 573 if (error == 0) 574 getmicrotime(&rpipe->pipe_atime); 575 pipeunlock(rpipe); 576 577 unlocked_error: 578 --rpipe->pipe_busy; 579 if (rpipe->pipe_busy == 0) { 580 cv_broadcast(&rpipe->pipe_draincv); 581 } 582 if (bp->cnt < MINPIPESIZE) { 583 cv_broadcast(&rpipe->pipe_wcv); 584 } 585 586 /* 587 * If anything was read off the buffer, signal to the writer it's 588 * possible to write more data. Also send signal if we are here for the 589 * first time after last write. 590 */ 591 if ((bp->size - bp->cnt) >= PIPE_BUF 592 && (ocnt != bp->cnt || (rpipe->pipe_state & PIPE_SIGNALR))) { 593 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 594 rpipe->pipe_state &= ~PIPE_SIGNALR; 595 } 596 597 mutex_exit(lock); 598 return (error); 599 } 600 601 #ifndef PIPE_NODIRECT 602 /* 603 * Allocate structure for loan transfer. 604 */ 605 static int 606 pipe_loan_alloc(struct pipe *wpipe, int npages) 607 { 608 vsize_t len; 609 610 len = (vsize_t)npages << PAGE_SHIFT; 611 atomic_add_int(&amountpipekva, len); 612 wpipe->pipe_map.kva = uvm_km_alloc(kernel_map, len, 0, 613 UVM_KMF_VAONLY | UVM_KMF_WAITVA); 614 if (wpipe->pipe_map.kva == 0) { 615 atomic_add_int(&amountpipekva, -len); 616 return (ENOMEM); 617 } 618 619 wpipe->pipe_map.npages = npages; 620 wpipe->pipe_map.pgs = kmem_alloc(npages * sizeof(struct vm_page *), 621 KM_SLEEP); 622 return (0); 623 } 624 625 /* 626 * Free resources allocated for loan transfer. 627 */ 628 static void 629 pipe_loan_free(struct pipe *wpipe) 630 { 631 vsize_t len; 632 633 len = (vsize_t)wpipe->pipe_map.npages << PAGE_SHIFT; 634 uvm_km_free(kernel_map, wpipe->pipe_map.kva, len, UVM_KMF_VAONLY); 635 wpipe->pipe_map.kva = 0; 636 atomic_add_int(&amountpipekva, -len); 637 kmem_free(wpipe->pipe_map.pgs, 638 wpipe->pipe_map.npages * sizeof(struct vm_page *)); 639 wpipe->pipe_map.pgs = NULL; 640 } 641 642 /* 643 * NetBSD direct write, using uvm_loan() mechanism. 644 * This implements the pipe buffer write mechanism. Note that only 645 * a direct write OR a normal pipe write can be pending at any given time. 646 * If there are any characters in the pipe buffer, the direct write will 647 * be deferred until the receiving process grabs all of the bytes from 648 * the pipe buffer. Then the direct mapping write is set-up. 649 * 650 * Called with the long-term pipe lock held. 651 */ 652 static int 653 pipe_direct_write(struct file *fp, struct pipe *wpipe, struct uio *uio) 654 { 655 int error, npages, j; 656 struct vm_page **pgs; 657 vaddr_t bbase, kva, base, bend; 658 vsize_t blen, bcnt; 659 voff_t bpos; 660 kmutex_t *lock = wpipe->pipe_lock; 661 662 KASSERT(mutex_owned(wpipe->pipe_lock)); 663 KASSERT(wpipe->pipe_map.cnt == 0); 664 665 mutex_exit(lock); 666 667 /* 668 * Handle first PIPE_CHUNK_SIZE bytes of buffer. Deal with buffers 669 * not aligned to PAGE_SIZE. 670 */ 671 bbase = (vaddr_t)uio->uio_iov->iov_base; 672 base = trunc_page(bbase); 673 bend = round_page(bbase + uio->uio_iov->iov_len); 674 blen = bend - base; 675 bpos = bbase - base; 676 677 if (blen > PIPE_DIRECT_CHUNK) { 678 blen = PIPE_DIRECT_CHUNK; 679 bend = base + blen; 680 bcnt = PIPE_DIRECT_CHUNK - bpos; 681 } else { 682 bcnt = uio->uio_iov->iov_len; 683 } 684 npages = blen >> PAGE_SHIFT; 685 686 /* 687 * Free the old kva if we need more pages than we have 688 * allocated. 689 */ 690 if (wpipe->pipe_map.kva != 0 && npages > wpipe->pipe_map.npages) 691 pipe_loan_free(wpipe); 692 693 /* Allocate new kva. */ 694 if (wpipe->pipe_map.kva == 0) { 695 error = pipe_loan_alloc(wpipe, npages); 696 if (error) { 697 mutex_enter(lock); 698 return (error); 699 } 700 } 701 702 /* Loan the write buffer memory from writer process */ 703 pgs = wpipe->pipe_map.pgs; 704 error = uvm_loan(&uio->uio_vmspace->vm_map, base, blen, 705 pgs, UVM_LOAN_TOPAGE); 706 if (error) { 707 pipe_loan_free(wpipe); 708 mutex_enter(lock); 709 return (ENOMEM); /* so that caller fallback to ordinary write */ 710 } 711 712 /* Enter the loaned pages to kva */ 713 kva = wpipe->pipe_map.kva; 714 for (j = 0; j < npages; j++, kva += PAGE_SIZE) { 715 pmap_kenter_pa(kva, VM_PAGE_TO_PHYS(pgs[j]), VM_PROT_READ); 716 } 717 pmap_update(pmap_kernel()); 718 719 /* Now we can put the pipe in direct write mode */ 720 wpipe->pipe_map.pos = bpos; 721 wpipe->pipe_map.cnt = bcnt; 722 723 /* 724 * But before we can let someone do a direct read, we 725 * have to wait until the pipe is drained. Release the 726 * pipe lock while we wait. 727 */ 728 mutex_enter(lock); 729 wpipe->pipe_state |= PIPE_DIRECTW; 730 pipeunlock(wpipe); 731 732 while (error == 0 && wpipe->pipe_buffer.cnt > 0) { 733 cv_broadcast(&wpipe->pipe_rcv); 734 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 735 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 736 error = EPIPE; 737 } 738 739 /* Pipe is drained; next read will off the direct buffer */ 740 wpipe->pipe_state |= PIPE_DIRECTR; 741 742 /* Wait until the reader is done */ 743 while (error == 0 && (wpipe->pipe_state & PIPE_DIRECTR)) { 744 cv_broadcast(&wpipe->pipe_rcv); 745 pipeselwakeup(wpipe, wpipe, POLL_IN); 746 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 747 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 748 error = EPIPE; 749 } 750 751 /* Take pipe out of direct write mode */ 752 wpipe->pipe_state &= ~(PIPE_DIRECTW | PIPE_DIRECTR); 753 754 /* Acquire the pipe lock and cleanup */ 755 (void)pipelock(wpipe, 0); 756 mutex_exit(lock); 757 758 if (pgs != NULL) { 759 pmap_kremove(wpipe->pipe_map.kva, blen); 760 pmap_update(pmap_kernel()); 761 uvm_unloan(pgs, npages, UVM_LOAN_TOPAGE); 762 } 763 if (error || amountpipekva > maxpipekva) 764 pipe_loan_free(wpipe); 765 766 mutex_enter(lock); 767 if (error) { 768 pipeselwakeup(wpipe, wpipe, POLL_ERR); 769 770 /* 771 * If nothing was read from what we offered, return error 772 * straight on. Otherwise update uio resid first. Caller 773 * will deal with the error condition, returning short 774 * write, error, or restarting the write(2) as appropriate. 775 */ 776 if (wpipe->pipe_map.cnt == bcnt) { 777 wpipe->pipe_map.cnt = 0; 778 cv_broadcast(&wpipe->pipe_wcv); 779 return (error); 780 } 781 782 bcnt -= wpipe->pipe_map.cnt; 783 } 784 785 uio->uio_resid -= bcnt; 786 /* uio_offset not updated, not set/used for write(2) */ 787 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + bcnt; 788 uio->uio_iov->iov_len -= bcnt; 789 if (uio->uio_iov->iov_len == 0) { 790 uio->uio_iov++; 791 uio->uio_iovcnt--; 792 } 793 794 wpipe->pipe_map.cnt = 0; 795 return (error); 796 } 797 #endif /* !PIPE_NODIRECT */ 798 799 static int 800 pipe_write(struct file *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 801 int flags) 802 { 803 struct pipe *wpipe, *rpipe; 804 struct pipebuf *bp; 805 kmutex_t *lock; 806 int error; 807 808 /* We want to write to our peer */ 809 rpipe = (struct pipe *) fp->f_data; 810 lock = rpipe->pipe_lock; 811 error = 0; 812 813 mutex_enter(lock); 814 wpipe = rpipe->pipe_peer; 815 816 /* 817 * Detect loss of pipe read side, issue SIGPIPE if lost. 818 */ 819 if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF) != 0) { 820 mutex_exit(lock); 821 return EPIPE; 822 } 823 ++wpipe->pipe_busy; 824 825 /* Aquire the long-term pipe lock */ 826 if ((error = pipelock(wpipe, 1)) != 0) { 827 --wpipe->pipe_busy; 828 if (wpipe->pipe_busy == 0) { 829 cv_broadcast(&wpipe->pipe_draincv); 830 } 831 mutex_exit(lock); 832 return (error); 833 } 834 835 bp = &wpipe->pipe_buffer; 836 837 /* 838 * If it is advantageous to resize the pipe buffer, do so. 839 */ 840 if ((uio->uio_resid > PIPE_SIZE) && 841 (nbigpipe < maxbigpipes) && 842 #ifndef PIPE_NODIRECT 843 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 844 #endif 845 (bp->size <= PIPE_SIZE) && (bp->cnt == 0)) { 846 847 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) 848 atomic_inc_uint(&nbigpipe); 849 } 850 851 while (uio->uio_resid) { 852 size_t space; 853 854 #ifndef PIPE_NODIRECT 855 /* 856 * Pipe buffered writes cannot be coincidental with 857 * direct writes. Also, only one direct write can be 858 * in progress at any one time. We wait until the currently 859 * executing direct write is completed before continuing. 860 * 861 * We break out if a signal occurs or the reader goes away. 862 */ 863 while (error == 0 && wpipe->pipe_state & PIPE_DIRECTW) { 864 cv_broadcast(&wpipe->pipe_rcv); 865 pipeunlock(wpipe); 866 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 867 (void)pipelock(wpipe, 0); 868 if (wpipe->pipe_state & PIPE_EOF) 869 error = EPIPE; 870 } 871 if (error) 872 break; 873 874 /* 875 * If the transfer is large, we can gain performance if 876 * we do process-to-process copies directly. 877 * If the write is non-blocking, we don't use the 878 * direct write mechanism. 879 * 880 * The direct write mechanism will detect the reader going 881 * away on us. 882 */ 883 if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) && 884 (fp->f_flag & FNONBLOCK) == 0 && 885 (wpipe->pipe_map.kva || (amountpipekva < limitpipekva))) { 886 error = pipe_direct_write(fp, wpipe, uio); 887 888 /* 889 * Break out if error occurred, unless it's ENOMEM. 890 * ENOMEM means we failed to allocate some resources 891 * for direct write, so we just fallback to ordinary 892 * write. If the direct write was successful, 893 * process rest of data via ordinary write. 894 */ 895 if (error == 0) 896 continue; 897 898 if (error != ENOMEM) 899 break; 900 } 901 #endif /* PIPE_NODIRECT */ 902 903 space = bp->size - bp->cnt; 904 905 /* Writes of size <= PIPE_BUF must be atomic. */ 906 if ((space < uio->uio_resid) && (uio->uio_resid <= PIPE_BUF)) 907 space = 0; 908 909 if (space > 0) { 910 int size; /* Transfer size */ 911 int segsize; /* first segment to transfer */ 912 913 /* 914 * Transfer size is minimum of uio transfer 915 * and free space in pipe buffer. 916 */ 917 if (space > uio->uio_resid) 918 size = uio->uio_resid; 919 else 920 size = space; 921 /* 922 * First segment to transfer is minimum of 923 * transfer size and contiguous space in 924 * pipe buffer. If first segment to transfer 925 * is less than the transfer size, we've got 926 * a wraparound in the buffer. 927 */ 928 segsize = bp->size - bp->in; 929 if (segsize > size) 930 segsize = size; 931 932 /* Transfer first segment */ 933 mutex_exit(lock); 934 error = uiomove((char *)bp->buffer + bp->in, segsize, 935 uio); 936 937 if (error == 0 && segsize < size) { 938 /* 939 * Transfer remaining part now, to 940 * support atomic writes. Wraparound 941 * happened. 942 */ 943 #ifdef DEBUG 944 if (bp->in + segsize != bp->size) 945 panic("Expected pipe buffer wraparound disappeared"); 946 #endif 947 948 error = uiomove(bp->buffer, 949 size - segsize, uio); 950 } 951 mutex_enter(lock); 952 if (error) 953 break; 954 955 bp->in += size; 956 if (bp->in >= bp->size) { 957 #ifdef DEBUG 958 if (bp->in != size - segsize + bp->size) 959 panic("Expected wraparound bad"); 960 #endif 961 bp->in = size - segsize; 962 } 963 964 bp->cnt += size; 965 #ifdef DEBUG 966 if (bp->cnt > bp->size) 967 panic("Pipe buffer overflow"); 968 #endif 969 } else { 970 /* 971 * If the "read-side" has been blocked, wake it up now. 972 */ 973 cv_broadcast(&wpipe->pipe_rcv); 974 975 /* 976 * don't block on non-blocking I/O 977 */ 978 if (fp->f_flag & FNONBLOCK) { 979 error = EAGAIN; 980 break; 981 } 982 983 /* 984 * We have no more space and have something to offer, 985 * wake up select/poll. 986 */ 987 if (bp->cnt) 988 pipeselwakeup(wpipe, wpipe, POLL_OUT); 989 990 pipeunlock(wpipe); 991 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 992 (void)pipelock(wpipe, 0); 993 if (error != 0) 994 break; 995 /* 996 * If read side wants to go away, we just issue a signal 997 * to ourselves. 998 */ 999 if (wpipe->pipe_state & PIPE_EOF) { 1000 error = EPIPE; 1001 break; 1002 } 1003 } 1004 } 1005 1006 --wpipe->pipe_busy; 1007 if (wpipe->pipe_busy == 0) { 1008 cv_broadcast(&wpipe->pipe_draincv); 1009 } 1010 if (bp->cnt > 0) { 1011 cv_broadcast(&wpipe->pipe_rcv); 1012 } 1013 1014 /* 1015 * Don't return EPIPE if I/O was successful 1016 */ 1017 if (error == EPIPE && bp->cnt == 0 && uio->uio_resid == 0) 1018 error = 0; 1019 1020 if (error == 0) 1021 getmicrotime(&wpipe->pipe_mtime); 1022 1023 /* 1024 * We have something to offer, wake up select/poll. 1025 * wpipe->pipe_map.cnt is always 0 in this point (direct write 1026 * is only done synchronously), so check only wpipe->pipe_buffer.cnt 1027 */ 1028 if (bp->cnt) 1029 pipeselwakeup(wpipe, wpipe, POLL_OUT); 1030 1031 /* 1032 * Arrange for next read(2) to do a signal. 1033 */ 1034 wpipe->pipe_state |= PIPE_SIGNALR; 1035 1036 pipeunlock(wpipe); 1037 mutex_exit(lock); 1038 return (error); 1039 } 1040 1041 /* 1042 * we implement a very minimal set of ioctls for compatibility with sockets. 1043 */ 1044 int 1045 pipe_ioctl(struct file *fp, u_long cmd, void *data) 1046 { 1047 struct pipe *pipe = fp->f_data; 1048 kmutex_t *lock = pipe->pipe_lock; 1049 1050 switch (cmd) { 1051 1052 case FIONBIO: 1053 return (0); 1054 1055 case FIOASYNC: 1056 mutex_enter(lock); 1057 if (*(int *)data) { 1058 pipe->pipe_state |= PIPE_ASYNC; 1059 } else { 1060 pipe->pipe_state &= ~PIPE_ASYNC; 1061 } 1062 mutex_exit(lock); 1063 return (0); 1064 1065 case FIONREAD: 1066 mutex_enter(lock); 1067 #ifndef PIPE_NODIRECT 1068 if (pipe->pipe_state & PIPE_DIRECTW) 1069 *(int *)data = pipe->pipe_map.cnt; 1070 else 1071 #endif 1072 *(int *)data = pipe->pipe_buffer.cnt; 1073 mutex_exit(lock); 1074 return (0); 1075 1076 case FIONWRITE: 1077 /* Look at other side */ 1078 pipe = pipe->pipe_peer; 1079 mutex_enter(lock); 1080 #ifndef PIPE_NODIRECT 1081 if (pipe->pipe_state & PIPE_DIRECTW) 1082 *(int *)data = pipe->pipe_map.cnt; 1083 else 1084 #endif 1085 *(int *)data = pipe->pipe_buffer.cnt; 1086 mutex_exit(lock); 1087 return (0); 1088 1089 case FIONSPACE: 1090 /* Look at other side */ 1091 pipe = pipe->pipe_peer; 1092 mutex_enter(lock); 1093 #ifndef PIPE_NODIRECT 1094 /* 1095 * If we're in direct-mode, we don't really have a 1096 * send queue, and any other write will block. Thus 1097 * zero seems like the best answer. 1098 */ 1099 if (pipe->pipe_state & PIPE_DIRECTW) 1100 *(int *)data = 0; 1101 else 1102 #endif 1103 *(int *)data = pipe->pipe_buffer.size - 1104 pipe->pipe_buffer.cnt; 1105 mutex_exit(lock); 1106 return (0); 1107 1108 case TIOCSPGRP: 1109 case FIOSETOWN: 1110 return fsetown(&pipe->pipe_pgid, cmd, data); 1111 1112 case TIOCGPGRP: 1113 case FIOGETOWN: 1114 return fgetown(pipe->pipe_pgid, cmd, data); 1115 1116 } 1117 return (EPASSTHROUGH); 1118 } 1119 1120 int 1121 pipe_poll(struct file *fp, int events) 1122 { 1123 struct pipe *rpipe = fp->f_data; 1124 struct pipe *wpipe; 1125 int eof = 0; 1126 int revents = 0; 1127 1128 mutex_enter(rpipe->pipe_lock); 1129 wpipe = rpipe->pipe_peer; 1130 1131 if (events & (POLLIN | POLLRDNORM)) 1132 if ((rpipe->pipe_buffer.cnt > 0) || 1133 #ifndef PIPE_NODIRECT 1134 (rpipe->pipe_state & PIPE_DIRECTR) || 1135 #endif 1136 (rpipe->pipe_state & PIPE_EOF)) 1137 revents |= events & (POLLIN | POLLRDNORM); 1138 1139 eof |= (rpipe->pipe_state & PIPE_EOF); 1140 1141 if (wpipe == NULL) 1142 revents |= events & (POLLOUT | POLLWRNORM); 1143 else { 1144 if (events & (POLLOUT | POLLWRNORM)) 1145 if ((wpipe->pipe_state & PIPE_EOF) || ( 1146 #ifndef PIPE_NODIRECT 1147 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 1148 #endif 1149 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 1150 revents |= events & (POLLOUT | POLLWRNORM); 1151 1152 eof |= (wpipe->pipe_state & PIPE_EOF); 1153 } 1154 1155 if (wpipe == NULL || eof) 1156 revents |= POLLHUP; 1157 1158 if (revents == 0) { 1159 if (events & (POLLIN | POLLRDNORM)) 1160 selrecord(curlwp, &rpipe->pipe_sel); 1161 1162 if (events & (POLLOUT | POLLWRNORM)) 1163 selrecord(curlwp, &wpipe->pipe_sel); 1164 } 1165 mutex_exit(rpipe->pipe_lock); 1166 1167 return (revents); 1168 } 1169 1170 static int 1171 pipe_stat(struct file *fp, struct stat *ub) 1172 { 1173 struct pipe *pipe = fp->f_data; 1174 1175 memset((void *)ub, 0, sizeof(*ub)); 1176 ub->st_mode = S_IFIFO | S_IRUSR | S_IWUSR; 1177 ub->st_blksize = pipe->pipe_buffer.size; 1178 if (ub->st_blksize == 0 && pipe->pipe_peer) 1179 ub->st_blksize = pipe->pipe_peer->pipe_buffer.size; 1180 ub->st_size = pipe->pipe_buffer.cnt; 1181 ub->st_blocks = (ub->st_size) ? 1 : 0; 1182 TIMEVAL_TO_TIMESPEC(&pipe->pipe_atime, &ub->st_atimespec); 1183 TIMEVAL_TO_TIMESPEC(&pipe->pipe_mtime, &ub->st_mtimespec); 1184 TIMEVAL_TO_TIMESPEC(&pipe->pipe_ctime, &ub->st_ctimespec); 1185 ub->st_uid = kauth_cred_geteuid(fp->f_cred); 1186 ub->st_gid = kauth_cred_getegid(fp->f_cred); 1187 1188 /* 1189 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 1190 * XXX (st_dev, st_ino) should be unique. 1191 */ 1192 return (0); 1193 } 1194 1195 /* ARGSUSED */ 1196 static int 1197 pipe_close(struct file *fp) 1198 { 1199 struct pipe *pipe = fp->f_data; 1200 1201 fp->f_data = NULL; 1202 pipeclose(fp, pipe); 1203 return (0); 1204 } 1205 1206 static void 1207 pipe_free_kmem(struct pipe *pipe) 1208 { 1209 1210 if (pipe->pipe_buffer.buffer != NULL) { 1211 if (pipe->pipe_buffer.size > PIPE_SIZE) 1212 atomic_dec_uint(&nbigpipe); 1213 uvm_km_free(kernel_map, 1214 (vaddr_t)pipe->pipe_buffer.buffer, 1215 pipe->pipe_buffer.size, UVM_KMF_PAGEABLE); 1216 atomic_add_int(&amountpipekva, -pipe->pipe_buffer.size); 1217 pipe->pipe_buffer.buffer = NULL; 1218 } 1219 #ifndef PIPE_NODIRECT 1220 if (pipe->pipe_map.kva != 0) { 1221 pipe_loan_free(pipe); 1222 pipe->pipe_map.cnt = 0; 1223 pipe->pipe_map.kva = 0; 1224 pipe->pipe_map.pos = 0; 1225 pipe->pipe_map.npages = 0; 1226 } 1227 #endif /* !PIPE_NODIRECT */ 1228 } 1229 1230 /* 1231 * shutdown the pipe 1232 */ 1233 static void 1234 pipeclose(struct file *fp, struct pipe *pipe) 1235 { 1236 struct pipe_mutex *mutex; 1237 kmutex_t *lock; 1238 struct pipe *ppipe; 1239 u_int refcnt; 1240 1241 if (pipe == NULL) 1242 return; 1243 1244 KASSERT(cv_is_valid(&pipe->pipe_rcv)); 1245 KASSERT(cv_is_valid(&pipe->pipe_wcv)); 1246 KASSERT(cv_is_valid(&pipe->pipe_draincv)); 1247 KASSERT(cv_is_valid(&pipe->pipe_lkcv)); 1248 1249 lock = pipe->pipe_lock; 1250 mutex_enter(lock); 1251 pipeselwakeup(pipe, pipe, POLL_HUP); 1252 1253 /* 1254 * If the other side is blocked, wake it up saying that 1255 * we want to close it down. 1256 */ 1257 pipe->pipe_state |= PIPE_EOF; 1258 if (pipe->pipe_busy) { 1259 while (pipe->pipe_busy) { 1260 cv_broadcast(&pipe->pipe_wcv); 1261 cv_wait_sig(&pipe->pipe_draincv, lock); 1262 } 1263 } 1264 1265 /* 1266 * Disconnect from peer 1267 */ 1268 if ((ppipe = pipe->pipe_peer) != NULL) { 1269 pipeselwakeup(ppipe, ppipe, POLL_HUP); 1270 ppipe->pipe_state |= PIPE_EOF; 1271 cv_broadcast(&ppipe->pipe_rcv); 1272 ppipe->pipe_peer = NULL; 1273 } 1274 1275 KASSERT((pipe->pipe_state & PIPE_LOCKFL) == 0); 1276 1277 mutex = (struct pipe_mutex *)lock; 1278 refcnt = --(mutex->pm_refcnt); 1279 KASSERT(refcnt == 0 || refcnt == 1); 1280 mutex_exit(lock); 1281 1282 /* 1283 * free resources 1284 */ 1285 pipe_free_kmem(pipe); 1286 cv_destroy(&pipe->pipe_rcv); 1287 cv_destroy(&pipe->pipe_wcv); 1288 cv_destroy(&pipe->pipe_draincv); 1289 cv_destroy(&pipe->pipe_lkcv); 1290 seldestroy(&pipe->pipe_sel); 1291 pool_cache_put(pipe_cache, pipe); 1292 if (refcnt == 0) 1293 pool_cache_put(pipe_mutex_cache, mutex); 1294 } 1295 1296 static void 1297 filt_pipedetach(struct knote *kn) 1298 { 1299 struct pipe *pipe; 1300 kmutex_t *lock; 1301 1302 pipe = ((file_t *)kn->kn_obj)->f_data; 1303 lock = pipe->pipe_lock; 1304 1305 mutex_enter(lock); 1306 1307 switch(kn->kn_filter) { 1308 case EVFILT_WRITE: 1309 /* need the peer structure, not our own */ 1310 pipe = pipe->pipe_peer; 1311 1312 /* if reader end already closed, just return */ 1313 if (pipe == NULL) { 1314 mutex_exit(lock); 1315 return; 1316 } 1317 1318 break; 1319 default: 1320 /* nothing to do */ 1321 break; 1322 } 1323 1324 #ifdef DIAGNOSTIC 1325 if (kn->kn_hook != pipe) 1326 panic("filt_pipedetach: inconsistent knote"); 1327 #endif 1328 1329 SLIST_REMOVE(&pipe->pipe_sel.sel_klist, kn, knote, kn_selnext); 1330 mutex_exit(lock); 1331 } 1332 1333 /*ARGSUSED*/ 1334 static int 1335 filt_piperead(struct knote *kn, long hint) 1336 { 1337 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_data; 1338 struct pipe *wpipe; 1339 1340 if ((hint & NOTE_SUBMIT) == 0) { 1341 mutex_enter(rpipe->pipe_lock); 1342 } 1343 wpipe = rpipe->pipe_peer; 1344 kn->kn_data = rpipe->pipe_buffer.cnt; 1345 1346 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1347 kn->kn_data = rpipe->pipe_map.cnt; 1348 1349 if ((rpipe->pipe_state & PIPE_EOF) || 1350 (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1351 kn->kn_flags |= EV_EOF; 1352 if ((hint & NOTE_SUBMIT) == 0) { 1353 mutex_exit(rpipe->pipe_lock); 1354 } 1355 return (1); 1356 } 1357 1358 if ((hint & NOTE_SUBMIT) == 0) { 1359 mutex_exit(rpipe->pipe_lock); 1360 } 1361 return (kn->kn_data > 0); 1362 } 1363 1364 /*ARGSUSED*/ 1365 static int 1366 filt_pipewrite(struct knote *kn, long hint) 1367 { 1368 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_data; 1369 struct pipe *wpipe; 1370 1371 if ((hint & NOTE_SUBMIT) == 0) { 1372 mutex_enter(rpipe->pipe_lock); 1373 } 1374 wpipe = rpipe->pipe_peer; 1375 1376 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1377 kn->kn_data = 0; 1378 kn->kn_flags |= EV_EOF; 1379 if ((hint & NOTE_SUBMIT) == 0) { 1380 mutex_exit(rpipe->pipe_lock); 1381 } 1382 return (1); 1383 } 1384 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1385 if (wpipe->pipe_state & PIPE_DIRECTW) 1386 kn->kn_data = 0; 1387 1388 if ((hint & NOTE_SUBMIT) == 0) { 1389 mutex_exit(rpipe->pipe_lock); 1390 } 1391 return (kn->kn_data >= PIPE_BUF); 1392 } 1393 1394 static const struct filterops pipe_rfiltops = 1395 { 1, NULL, filt_pipedetach, filt_piperead }; 1396 static const struct filterops pipe_wfiltops = 1397 { 1, NULL, filt_pipedetach, filt_pipewrite }; 1398 1399 /*ARGSUSED*/ 1400 static int 1401 pipe_kqfilter(struct file *fp, struct knote *kn) 1402 { 1403 struct pipe *pipe; 1404 kmutex_t *lock; 1405 1406 pipe = ((file_t *)kn->kn_obj)->f_data; 1407 lock = pipe->pipe_lock; 1408 1409 mutex_enter(lock); 1410 1411 switch (kn->kn_filter) { 1412 case EVFILT_READ: 1413 kn->kn_fop = &pipe_rfiltops; 1414 break; 1415 case EVFILT_WRITE: 1416 kn->kn_fop = &pipe_wfiltops; 1417 pipe = pipe->pipe_peer; 1418 if (pipe == NULL) { 1419 /* other end of pipe has been closed */ 1420 mutex_exit(lock); 1421 return (EBADF); 1422 } 1423 break; 1424 default: 1425 mutex_exit(lock); 1426 return (EINVAL); 1427 } 1428 1429 kn->kn_hook = pipe; 1430 SLIST_INSERT_HEAD(&pipe->pipe_sel.sel_klist, kn, kn_selnext); 1431 mutex_exit(lock); 1432 1433 return (0); 1434 } 1435 1436 /* 1437 * Handle pipe sysctls. 1438 */ 1439 SYSCTL_SETUP(sysctl_kern_pipe_setup, "sysctl kern.pipe subtree setup") 1440 { 1441 1442 sysctl_createv(clog, 0, NULL, NULL, 1443 CTLFLAG_PERMANENT, 1444 CTLTYPE_NODE, "kern", NULL, 1445 NULL, 0, NULL, 0, 1446 CTL_KERN, CTL_EOL); 1447 sysctl_createv(clog, 0, NULL, NULL, 1448 CTLFLAG_PERMANENT, 1449 CTLTYPE_NODE, "pipe", 1450 SYSCTL_DESCR("Pipe settings"), 1451 NULL, 0, NULL, 0, 1452 CTL_KERN, KERN_PIPE, CTL_EOL); 1453 1454 sysctl_createv(clog, 0, NULL, NULL, 1455 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1456 CTLTYPE_INT, "maxkvasz", 1457 SYSCTL_DESCR("Maximum amount of kernel memory to be " 1458 "used for pipes"), 1459 NULL, 0, &maxpipekva, 0, 1460 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXKVASZ, CTL_EOL); 1461 sysctl_createv(clog, 0, NULL, NULL, 1462 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1463 CTLTYPE_INT, "maxloankvasz", 1464 SYSCTL_DESCR("Limit for direct transfers via page loan"), 1465 NULL, 0, &limitpipekva, 0, 1466 CTL_KERN, KERN_PIPE, KERN_PIPE_LIMITKVA, CTL_EOL); 1467 sysctl_createv(clog, 0, NULL, NULL, 1468 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1469 CTLTYPE_INT, "maxbigpipes", 1470 SYSCTL_DESCR("Maximum number of \"big\" pipes"), 1471 NULL, 0, &maxbigpipes, 0, 1472 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXBIGPIPES, CTL_EOL); 1473 sysctl_createv(clog, 0, NULL, NULL, 1474 CTLFLAG_PERMANENT, 1475 CTLTYPE_INT, "nbigpipes", 1476 SYSCTL_DESCR("Number of \"big\" pipes"), 1477 NULL, 0, &nbigpipe, 0, 1478 CTL_KERN, KERN_PIPE, KERN_PIPE_NBIGPIPES, CTL_EOL); 1479 sysctl_createv(clog, 0, NULL, NULL, 1480 CTLFLAG_PERMANENT, 1481 CTLTYPE_INT, "kvasize", 1482 SYSCTL_DESCR("Amount of kernel memory consumed by pipe " 1483 "buffers"), 1484 NULL, 0, &amountpipekva, 0, 1485 CTL_KERN, KERN_PIPE, KERN_PIPE_KVASIZE, CTL_EOL); 1486 } 1487