1 /* $NetBSD: sys_pipe.c,v 1.85 2007/07/09 21:10:56 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2003, 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Paul Kranenburg, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Copyright (c) 1996 John S. Dyson 41 * All rights reserved. 42 * 43 * Redistribution and use in source and binary forms, with or without 44 * modification, are permitted provided that the following conditions 45 * are met: 46 * 1. Redistributions of source code must retain the above copyright 47 * notice immediately at the beginning of the file, without modification, 48 * this list of conditions, and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. Absolutely no warranty of function or purpose is made by the author 53 * John S. Dyson. 54 * 4. Modifications may be freely made to this file if the above conditions 55 * are met. 56 * 57 * $FreeBSD: src/sys/kern/sys_pipe.c,v 1.95 2002/03/09 22:06:31 alfred Exp $ 58 */ 59 60 /* 61 * This file contains a high-performance replacement for the socket-based 62 * pipes scheme originally used in FreeBSD/4.4Lite. It does not support 63 * all features of sockets, but does do everything that pipes normally 64 * do. 65 * 66 * Adaption for NetBSD UVM, including uvm_loan() based direct write, was 67 * written by Jaromir Dolecek. 68 */ 69 70 /* 71 * This code has two modes of operation, a small write mode and a large 72 * write mode. The small write mode acts like conventional pipes with 73 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 74 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 75 * and PIPE_SIZE in size it is mapped read-only into the kernel address space 76 * using the UVM page loan facility from where the receiving process can copy 77 * the data directly from the pages in the sending process. 78 * 79 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 80 * happen for small transfers so that the system will not spend all of 81 * its time context switching. PIPE_SIZE is constrained by the 82 * amount of kernel virtual memory. 83 */ 84 85 #include <sys/cdefs.h> 86 __KERNEL_RCSID(0, "$NetBSD: sys_pipe.c,v 1.85 2007/07/09 21:10:56 ad Exp $"); 87 88 #include <sys/param.h> 89 #include <sys/systm.h> 90 #include <sys/proc.h> 91 #include <sys/fcntl.h> 92 #include <sys/file.h> 93 #include <sys/filedesc.h> 94 #include <sys/filio.h> 95 #include <sys/kernel.h> 96 #include <sys/ttycom.h> 97 #include <sys/stat.h> 98 #include <sys/malloc.h> 99 #include <sys/poll.h> 100 #include <sys/signalvar.h> 101 #include <sys/vnode.h> 102 #include <sys/uio.h> 103 #include <sys/lock.h> 104 #include <sys/select.h> 105 #include <sys/mount.h> 106 #include <sys/syscallargs.h> 107 #include <uvm/uvm.h> 108 #include <sys/sysctl.h> 109 #include <sys/kauth.h> 110 111 #include <sys/pipe.h> 112 113 /* 114 * Use this define if you want to disable *fancy* VM things. Expect an 115 * approx 30% decrease in transfer rate. 116 */ 117 /* #define PIPE_NODIRECT */ 118 119 /* 120 * interfaces to the outside world 121 */ 122 static int pipe_read(struct file *fp, off_t *offset, struct uio *uio, 123 kauth_cred_t cred, int flags); 124 static int pipe_write(struct file *fp, off_t *offset, struct uio *uio, 125 kauth_cred_t cred, int flags); 126 static int pipe_close(struct file *fp, struct lwp *l); 127 static int pipe_poll(struct file *fp, int events, struct lwp *l); 128 static int pipe_kqfilter(struct file *fp, struct knote *kn); 129 static int pipe_stat(struct file *fp, struct stat *sb, struct lwp *l); 130 static int pipe_ioctl(struct file *fp, u_long cmd, void *data, 131 struct lwp *l); 132 133 static const struct fileops pipeops = { 134 pipe_read, pipe_write, pipe_ioctl, fnullop_fcntl, pipe_poll, 135 pipe_stat, pipe_close, pipe_kqfilter 136 }; 137 138 /* 139 * Default pipe buffer size(s), this can be kind-of large now because pipe 140 * space is pageable. The pipe code will try to maintain locality of 141 * reference for performance reasons, so small amounts of outstanding I/O 142 * will not wipe the cache. 143 */ 144 #define MINPIPESIZE (PIPE_SIZE/3) 145 #define MAXPIPESIZE (2*PIPE_SIZE/3) 146 147 /* 148 * Maximum amount of kva for pipes -- this is kind-of a soft limit, but 149 * is there so that on large systems, we don't exhaust it. 150 */ 151 #define MAXPIPEKVA (8*1024*1024) 152 static int maxpipekva = MAXPIPEKVA; 153 154 /* 155 * Limit for direct transfers, we cannot, of course limit 156 * the amount of kva for pipes in general though. 157 */ 158 #define LIMITPIPEKVA (16*1024*1024) 159 static int limitpipekva = LIMITPIPEKVA; 160 161 /* 162 * Limit the number of "big" pipes 163 */ 164 #define LIMITBIGPIPES 32 165 static int maxbigpipes = LIMITBIGPIPES; 166 static int nbigpipe = 0; 167 168 /* 169 * Amount of KVA consumed by pipe buffers. 170 */ 171 static int amountpipekva = 0; 172 173 MALLOC_DEFINE(M_PIPE, "pipe", "Pipe structures"); 174 175 static void pipeclose(struct file *fp, struct pipe *pipe); 176 static void pipe_free_kmem(struct pipe *pipe); 177 static int pipe_create(struct pipe **pipep, int allockva); 178 static int pipelock(struct pipe *pipe, int catch); 179 static inline void pipeunlock(struct pipe *pipe); 180 static void pipeselwakeup(struct pipe *pipe, struct pipe *sigp, int code); 181 #ifndef PIPE_NODIRECT 182 static int pipe_direct_write(struct file *fp, struct pipe *wpipe, 183 struct uio *uio); 184 #endif 185 static int pipespace(struct pipe *pipe, int size); 186 187 #ifndef PIPE_NODIRECT 188 static int pipe_loan_alloc(struct pipe *, int); 189 static void pipe_loan_free(struct pipe *); 190 #endif /* PIPE_NODIRECT */ 191 192 static POOL_INIT(pipe_pool, sizeof(struct pipe), 0, 0, 0, "pipepl", 193 &pool_allocator_nointr, IPL_NONE); 194 195 static krwlock_t pipe_peer_lock; 196 197 void 198 pipe_init(void) 199 { 200 201 rw_init(&pipe_peer_lock); 202 } 203 204 /* 205 * The pipe system call for the DTYPE_PIPE type of pipes 206 */ 207 208 /* ARGSUSED */ 209 int 210 sys_pipe(struct lwp *l, void *v, register_t *retval) 211 { 212 struct file *rf, *wf; 213 struct pipe *rpipe, *wpipe; 214 int fd, error; 215 216 rpipe = wpipe = NULL; 217 if (pipe_create(&rpipe, 1) || pipe_create(&wpipe, 0)) { 218 pipeclose(NULL, rpipe); 219 pipeclose(NULL, wpipe); 220 return (ENFILE); 221 } 222 223 /* 224 * Note: the file structure returned from falloc() is marked 225 * as 'larval' initially. Unless we mark it as 'mature' by 226 * FILE_SET_MATURE(), any attempt to do anything with it would 227 * return EBADF, including e.g. dup(2) or close(2). This avoids 228 * file descriptor races if we block in the second falloc(). 229 */ 230 231 error = falloc(l, &rf, &fd); 232 if (error) 233 goto free2; 234 retval[0] = fd; 235 rf->f_flag = FREAD; 236 rf->f_type = DTYPE_PIPE; 237 rf->f_data = (void *)rpipe; 238 rf->f_ops = &pipeops; 239 240 error = falloc(l, &wf, &fd); 241 if (error) 242 goto free3; 243 retval[1] = fd; 244 wf->f_flag = FWRITE; 245 wf->f_type = DTYPE_PIPE; 246 wf->f_data = (void *)wpipe; 247 wf->f_ops = &pipeops; 248 249 rpipe->pipe_peer = wpipe; 250 wpipe->pipe_peer = rpipe; 251 252 FILE_SET_MATURE(rf); 253 FILE_SET_MATURE(wf); 254 FILE_UNUSE(rf, l); 255 FILE_UNUSE(wf, l); 256 return (0); 257 free3: 258 FILE_UNUSE(rf, l); 259 ffree(rf); 260 fdremove(l->l_proc->p_fd, retval[0]); 261 free2: 262 pipeclose(NULL, wpipe); 263 pipeclose(NULL, rpipe); 264 265 return (error); 266 } 267 268 /* 269 * Allocate kva for pipe circular buffer, the space is pageable 270 * This routine will 'realloc' the size of a pipe safely, if it fails 271 * it will retain the old buffer. 272 * If it fails it will return ENOMEM. 273 */ 274 static int 275 pipespace(struct pipe *pipe, int size) 276 { 277 void *buffer; 278 /* 279 * Allocate pageable virtual address space. Physical memory is 280 * allocated on demand. 281 */ 282 buffer = (void *) uvm_km_alloc(kernel_map, round_page(size), 0, 283 UVM_KMF_PAGEABLE); 284 if (buffer == NULL) 285 return (ENOMEM); 286 287 /* free old resources if we're resizing */ 288 pipe_free_kmem(pipe); 289 pipe->pipe_buffer.buffer = buffer; 290 pipe->pipe_buffer.size = size; 291 pipe->pipe_buffer.in = 0; 292 pipe->pipe_buffer.out = 0; 293 pipe->pipe_buffer.cnt = 0; 294 amountpipekva += pipe->pipe_buffer.size; 295 return (0); 296 } 297 298 /* 299 * Initialize and allocate VM and memory for pipe. 300 */ 301 static int 302 pipe_create(struct pipe **pipep, int allockva) 303 { 304 struct pipe *pipe; 305 int error; 306 307 pipe = *pipep = pool_get(&pipe_pool, PR_WAITOK); 308 309 /* Initialize */ 310 memset(pipe, 0, sizeof(struct pipe)); 311 pipe->pipe_state = PIPE_SIGNALR; 312 313 getmicrotime(&pipe->pipe_ctime); 314 pipe->pipe_atime = pipe->pipe_ctime; 315 pipe->pipe_mtime = pipe->pipe_ctime; 316 mutex_init(&pipe->pipe_lock, MUTEX_DEFAULT, IPL_NONE); 317 cv_init(&pipe->pipe_cv, "pipe"); 318 cv_init(&pipe->pipe_lkcv, "pipelk"); 319 320 if (allockva && (error = pipespace(pipe, PIPE_SIZE))) 321 return (error); 322 323 return (0); 324 } 325 326 327 /* 328 * Lock a pipe for I/O, blocking other access 329 * Called with pipe spin lock held. 330 * Return with pipe spin lock released on success. 331 */ 332 static int 333 pipelock(struct pipe *pipe, int catch) 334 { 335 int error; 336 337 KASSERT(mutex_owned(&pipe->pipe_lock)); 338 339 while (pipe->pipe_state & PIPE_LOCKFL) { 340 pipe->pipe_state |= PIPE_LWANT; 341 if (catch) { 342 error = cv_wait_sig(&pipe->pipe_lkcv, 343 &pipe->pipe_lock); 344 if (error != 0) 345 return error; 346 } else 347 cv_wait(&pipe->pipe_lkcv, &pipe->pipe_lock); 348 } 349 350 pipe->pipe_state |= PIPE_LOCKFL; 351 mutex_exit(&pipe->pipe_lock); 352 353 return 0; 354 } 355 356 /* 357 * unlock a pipe I/O lock 358 */ 359 static inline void 360 pipeunlock(struct pipe *pipe) 361 { 362 363 KASSERT(pipe->pipe_state & PIPE_LOCKFL); 364 365 pipe->pipe_state &= ~PIPE_LOCKFL; 366 if (pipe->pipe_state & PIPE_LWANT) { 367 pipe->pipe_state &= ~PIPE_LWANT; 368 cv_broadcast(&pipe->pipe_lkcv); 369 } 370 } 371 372 /* 373 * Select/poll wakup. This also sends SIGIO to peer connected to 374 * 'sigpipe' side of pipe. 375 */ 376 static void 377 pipeselwakeup(struct pipe *selp, struct pipe *sigp, int code) 378 { 379 int band; 380 381 selnotify(&selp->pipe_sel, NOTE_SUBMIT); 382 383 if (sigp == NULL || (sigp->pipe_state & PIPE_ASYNC) == 0) 384 return; 385 386 switch (code) { 387 case POLL_IN: 388 band = POLLIN|POLLRDNORM; 389 break; 390 case POLL_OUT: 391 band = POLLOUT|POLLWRNORM; 392 break; 393 case POLL_HUP: 394 band = POLLHUP; 395 break; 396 #if POLL_HUP != POLL_ERR 397 case POLL_ERR: 398 band = POLLERR; 399 break; 400 #endif 401 default: 402 band = 0; 403 #ifdef DIAGNOSTIC 404 printf("bad siginfo code %d in pipe notification.\n", code); 405 #endif 406 break; 407 } 408 409 fownsignal(sigp->pipe_pgid, SIGIO, code, band, selp); 410 } 411 412 /* ARGSUSED */ 413 static int 414 pipe_read(struct file *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 415 int flags) 416 { 417 struct pipe *rpipe = (struct pipe *) fp->f_data; 418 struct pipebuf *bp = &rpipe->pipe_buffer; 419 int error; 420 size_t nread = 0; 421 size_t size; 422 size_t ocnt; 423 424 mutex_enter(&rpipe->pipe_lock); 425 ++rpipe->pipe_busy; 426 ocnt = bp->cnt; 427 428 again: 429 error = pipelock(rpipe, 1); 430 if (error) 431 goto unlocked_error; 432 433 while (uio->uio_resid) { 434 /* 435 * normal pipe buffer receive 436 */ 437 if (bp->cnt > 0) { 438 size = bp->size - bp->out; 439 if (size > bp->cnt) 440 size = bp->cnt; 441 if (size > uio->uio_resid) 442 size = uio->uio_resid; 443 444 error = uiomove((char *)bp->buffer + bp->out, size, uio); 445 if (error) 446 break; 447 448 bp->out += size; 449 if (bp->out >= bp->size) 450 bp->out = 0; 451 452 bp->cnt -= size; 453 454 /* 455 * If there is no more to read in the pipe, reset 456 * its pointers to the beginning. This improves 457 * cache hit stats. 458 */ 459 if (bp->cnt == 0) { 460 bp->in = 0; 461 bp->out = 0; 462 } 463 nread += size; 464 continue; 465 } 466 467 /* Lock to see up-to-date value of pipe_status. */ 468 mutex_enter(&rpipe->pipe_lock); 469 470 #ifndef PIPE_NODIRECT 471 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) { 472 /* 473 * Direct copy, bypassing a kernel buffer. 474 */ 475 void * va; 476 477 KASSERT(rpipe->pipe_state & PIPE_DIRECTW); 478 mutex_exit(&rpipe->pipe_lock); 479 480 size = rpipe->pipe_map.cnt; 481 if (size > uio->uio_resid) 482 size = uio->uio_resid; 483 484 va = (char *)rpipe->pipe_map.kva + rpipe->pipe_map.pos; 485 error = uiomove(va, size, uio); 486 if (error) 487 break; 488 nread += size; 489 rpipe->pipe_map.pos += size; 490 rpipe->pipe_map.cnt -= size; 491 if (rpipe->pipe_map.cnt == 0) { 492 mutex_enter(&rpipe->pipe_lock); 493 rpipe->pipe_state &= ~PIPE_DIRECTR; 494 cv_broadcast(&rpipe->pipe_cv); 495 mutex_exit(&rpipe->pipe_lock); 496 } 497 continue; 498 } 499 #endif 500 /* 501 * Break if some data was read. 502 */ 503 if (nread > 0) { 504 mutex_exit(&rpipe->pipe_lock); 505 break; 506 } 507 508 /* 509 * detect EOF condition 510 * read returns 0 on EOF, no need to set error 511 */ 512 if (rpipe->pipe_state & PIPE_EOF) { 513 mutex_exit(&rpipe->pipe_lock); 514 break; 515 } 516 517 /* 518 * don't block on non-blocking I/O 519 */ 520 if (fp->f_flag & FNONBLOCK) { 521 mutex_exit(&rpipe->pipe_lock); 522 error = EAGAIN; 523 break; 524 } 525 526 /* 527 * Unlock the pipe buffer for our remaining processing. 528 * We will either break out with an error or we will 529 * sleep and relock to loop. 530 */ 531 pipeunlock(rpipe); 532 533 /* 534 * Re-check to see if more direct writes are pending. 535 */ 536 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) 537 goto again; 538 539 /* 540 * We want to read more, wake up select/poll. 541 */ 542 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_IN); 543 544 /* 545 * If the "write-side" is blocked, wake it up now. 546 */ 547 if (rpipe->pipe_state & PIPE_WANTW) { 548 rpipe->pipe_state &= ~PIPE_WANTW; 549 cv_broadcast(&rpipe->pipe_cv); 550 } 551 552 /* Now wait until the pipe is filled */ 553 rpipe->pipe_state |= PIPE_WANTR; 554 error = cv_wait_sig(&rpipe->pipe_cv, &rpipe->pipe_lock); 555 if (error != 0) 556 goto unlocked_error; 557 goto again; 558 } 559 560 if (error == 0) 561 getmicrotime(&rpipe->pipe_atime); 562 563 mutex_enter(&rpipe->pipe_lock); 564 pipeunlock(rpipe); 565 566 unlocked_error: 567 --rpipe->pipe_busy; 568 569 /* 570 * PIPE_WANTCLOSE processing only makes sense if pipe_busy is 0. 571 */ 572 if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANTCLOSE)) { 573 rpipe->pipe_state &= ~(PIPE_WANTCLOSE|PIPE_WANTW); 574 cv_broadcast(&rpipe->pipe_cv); 575 } else if (bp->cnt < MINPIPESIZE) { 576 /* 577 * Handle write blocking hysteresis. 578 */ 579 if (rpipe->pipe_state & PIPE_WANTW) { 580 rpipe->pipe_state &= ~PIPE_WANTW; 581 cv_broadcast(&rpipe->pipe_cv); 582 } 583 } 584 585 /* 586 * If anything was read off the buffer, signal to the writer it's 587 * possible to write more data. Also send signal if we are here for the 588 * first time after last write. 589 */ 590 if ((bp->size - bp->cnt) >= PIPE_BUF 591 && (ocnt != bp->cnt || (rpipe->pipe_state & PIPE_SIGNALR))) { 592 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 593 rpipe->pipe_state &= ~PIPE_SIGNALR; 594 } 595 596 mutex_exit(&rpipe->pipe_lock); 597 return (error); 598 } 599 600 #ifndef PIPE_NODIRECT 601 /* 602 * Allocate structure for loan transfer. 603 */ 604 static int 605 pipe_loan_alloc(struct pipe *wpipe, int npages) 606 { 607 vsize_t len; 608 609 len = (vsize_t)npages << PAGE_SHIFT; 610 wpipe->pipe_map.kva = uvm_km_alloc(kernel_map, len, 0, 611 UVM_KMF_VAONLY | UVM_KMF_WAITVA); 612 if (wpipe->pipe_map.kva == 0) 613 return (ENOMEM); 614 615 amountpipekva += len; 616 wpipe->pipe_map.npages = npages; 617 wpipe->pipe_map.pgs = malloc(npages * sizeof(struct vm_page *), M_PIPE, 618 M_WAITOK); 619 return (0); 620 } 621 622 /* 623 * Free resources allocated for loan transfer. 624 */ 625 static void 626 pipe_loan_free(struct pipe *wpipe) 627 { 628 vsize_t len; 629 630 len = (vsize_t)wpipe->pipe_map.npages << PAGE_SHIFT; 631 uvm_km_free(kernel_map, wpipe->pipe_map.kva, len, UVM_KMF_VAONLY); 632 wpipe->pipe_map.kva = 0; 633 amountpipekva -= len; 634 free(wpipe->pipe_map.pgs, M_PIPE); 635 wpipe->pipe_map.pgs = NULL; 636 } 637 638 /* 639 * NetBSD direct write, using uvm_loan() mechanism. 640 * This implements the pipe buffer write mechanism. Note that only 641 * a direct write OR a normal pipe write can be pending at any given time. 642 * If there are any characters in the pipe buffer, the direct write will 643 * be deferred until the receiving process grabs all of the bytes from 644 * the pipe buffer. Then the direct mapping write is set-up. 645 * 646 * Called with the long-term pipe lock held. 647 */ 648 static int 649 pipe_direct_write(struct file *fp, struct pipe *wpipe, struct uio *uio) 650 { 651 int error, npages, j; 652 struct vm_page **pgs; 653 vaddr_t bbase, kva, base, bend; 654 vsize_t blen, bcnt; 655 voff_t bpos; 656 657 KASSERT(wpipe->pipe_map.cnt == 0); 658 659 /* 660 * Handle first PIPE_CHUNK_SIZE bytes of buffer. Deal with buffers 661 * not aligned to PAGE_SIZE. 662 */ 663 bbase = (vaddr_t)uio->uio_iov->iov_base; 664 base = trunc_page(bbase); 665 bend = round_page(bbase + uio->uio_iov->iov_len); 666 blen = bend - base; 667 bpos = bbase - base; 668 669 if (blen > PIPE_DIRECT_CHUNK) { 670 blen = PIPE_DIRECT_CHUNK; 671 bend = base + blen; 672 bcnt = PIPE_DIRECT_CHUNK - bpos; 673 } else { 674 bcnt = uio->uio_iov->iov_len; 675 } 676 npages = blen >> PAGE_SHIFT; 677 678 /* 679 * Free the old kva if we need more pages than we have 680 * allocated. 681 */ 682 if (wpipe->pipe_map.kva != 0 && npages > wpipe->pipe_map.npages) 683 pipe_loan_free(wpipe); 684 685 /* Allocate new kva. */ 686 if (wpipe->pipe_map.kva == 0) { 687 error = pipe_loan_alloc(wpipe, npages); 688 if (error) 689 return (error); 690 } 691 692 /* Loan the write buffer memory from writer process */ 693 pgs = wpipe->pipe_map.pgs; 694 error = uvm_loan(&uio->uio_vmspace->vm_map, base, blen, 695 pgs, UVM_LOAN_TOPAGE); 696 if (error) { 697 pipe_loan_free(wpipe); 698 return (ENOMEM); /* so that caller fallback to ordinary write */ 699 } 700 701 /* Enter the loaned pages to kva */ 702 kva = wpipe->pipe_map.kva; 703 for (j = 0; j < npages; j++, kva += PAGE_SIZE) { 704 pmap_kenter_pa(kva, VM_PAGE_TO_PHYS(pgs[j]), VM_PROT_READ); 705 } 706 pmap_update(pmap_kernel()); 707 708 /* Now we can put the pipe in direct write mode */ 709 wpipe->pipe_map.pos = bpos; 710 wpipe->pipe_map.cnt = bcnt; 711 712 /* 713 * But before we can let someone do a direct read, we 714 * have to wait until the pipe is drained. Release the 715 * pipe lock while we wait. 716 */ 717 mutex_enter(&wpipe->pipe_lock); 718 wpipe->pipe_state |= PIPE_DIRECTW; 719 pipeunlock(wpipe); 720 721 while (error == 0 && wpipe->pipe_buffer.cnt > 0) { 722 if (wpipe->pipe_state & PIPE_WANTR) { 723 wpipe->pipe_state &= ~PIPE_WANTR; 724 cv_broadcast(&wpipe->pipe_cv); 725 } 726 727 wpipe->pipe_state |= PIPE_WANTW; 728 error = cv_wait_sig(&wpipe->pipe_cv, &wpipe->pipe_lock); 729 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 730 error = EPIPE; 731 } 732 733 /* Pipe is drained; next read will off the direct buffer */ 734 wpipe->pipe_state |= PIPE_DIRECTR; 735 736 /* Wait until the reader is done */ 737 while (error == 0 && (wpipe->pipe_state & PIPE_DIRECTR)) { 738 if (wpipe->pipe_state & PIPE_WANTR) { 739 wpipe->pipe_state &= ~PIPE_WANTR; 740 cv_broadcast(&wpipe->pipe_cv); 741 } 742 pipeselwakeup(wpipe, wpipe, POLL_IN); 743 error = cv_wait_sig(&wpipe->pipe_cv, &wpipe->pipe_lock); 744 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 745 error = EPIPE; 746 } 747 748 /* Take pipe out of direct write mode */ 749 wpipe->pipe_state &= ~(PIPE_DIRECTW | PIPE_DIRECTR); 750 751 /* Acquire the pipe lock and cleanup */ 752 (void)pipelock(wpipe, 0); 753 754 if (pgs != NULL) { 755 pmap_kremove(wpipe->pipe_map.kva, blen); 756 uvm_unloan(pgs, npages, UVM_LOAN_TOPAGE); 757 } 758 if (error || amountpipekva > maxpipekva) 759 pipe_loan_free(wpipe); 760 761 if (error) { 762 pipeselwakeup(wpipe, wpipe, POLL_ERR); 763 764 /* 765 * If nothing was read from what we offered, return error 766 * straight on. Otherwise update uio resid first. Caller 767 * will deal with the error condition, returning short 768 * write, error, or restarting the write(2) as appropriate. 769 */ 770 if (wpipe->pipe_map.cnt == bcnt) { 771 wpipe->pipe_map.cnt = 0; 772 cv_broadcast(&wpipe->pipe_cv); 773 return (error); 774 } 775 776 bcnt -= wpipe->pipe_map.cnt; 777 } 778 779 uio->uio_resid -= bcnt; 780 /* uio_offset not updated, not set/used for write(2) */ 781 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + bcnt; 782 uio->uio_iov->iov_len -= bcnt; 783 if (uio->uio_iov->iov_len == 0) { 784 uio->uio_iov++; 785 uio->uio_iovcnt--; 786 } 787 788 wpipe->pipe_map.cnt = 0; 789 return (error); 790 } 791 #endif /* !PIPE_NODIRECT */ 792 793 static int 794 pipe_write(struct file *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 795 int flags) 796 { 797 struct pipe *wpipe, *rpipe; 798 struct pipebuf *bp; 799 int error; 800 801 /* We want to write to our peer */ 802 rpipe = (struct pipe *) fp->f_data; 803 804 retry: 805 error = 0; 806 mutex_enter(&rpipe->pipe_lock); 807 wpipe = rpipe->pipe_peer; 808 809 /* 810 * Detect loss of pipe read side, issue SIGPIPE if lost. 811 */ 812 if (wpipe == NULL) 813 error = EPIPE; 814 else if (mutex_tryenter(&wpipe->pipe_lock) == 0) { 815 /* Deal with race for peer */ 816 mutex_exit(&rpipe->pipe_lock); 817 /* XXX Might be about to deadlock w/kernel_lock. */ 818 yield(); 819 goto retry; 820 } else if ((wpipe->pipe_state & PIPE_EOF) != 0) { 821 mutex_exit(&wpipe->pipe_lock); 822 error = EPIPE; 823 } 824 825 mutex_exit(&rpipe->pipe_lock); 826 if (error != 0) 827 return (error); 828 829 ++wpipe->pipe_busy; 830 831 /* Aquire the long-term pipe lock */ 832 if ((error = pipelock(wpipe,1)) != 0) { 833 --wpipe->pipe_busy; 834 if (wpipe->pipe_busy == 0 835 && (wpipe->pipe_state & PIPE_WANTCLOSE)) { 836 wpipe->pipe_state &= ~(PIPE_WANTCLOSE | PIPE_WANTR); 837 cv_broadcast(&wpipe->pipe_cv); 838 } 839 mutex_exit(&wpipe->pipe_lock); 840 return (error); 841 } 842 843 bp = &wpipe->pipe_buffer; 844 845 /* 846 * If it is advantageous to resize the pipe buffer, do so. 847 */ 848 if ((uio->uio_resid > PIPE_SIZE) && 849 (nbigpipe < maxbigpipes) && 850 #ifndef PIPE_NODIRECT 851 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 852 #endif 853 (bp->size <= PIPE_SIZE) && (bp->cnt == 0)) { 854 855 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) 856 nbigpipe++; 857 } 858 859 while (uio->uio_resid) { 860 size_t space; 861 862 #ifndef PIPE_NODIRECT 863 /* 864 * Pipe buffered writes cannot be coincidental with 865 * direct writes. Also, only one direct write can be 866 * in progress at any one time. We wait until the currently 867 * executing direct write is completed before continuing. 868 * 869 * We break out if a signal occurs or the reader goes away. 870 */ 871 while (error == 0 && wpipe->pipe_state & PIPE_DIRECTW) { 872 mutex_enter(&wpipe->pipe_lock); 873 if (wpipe->pipe_state & PIPE_WANTR) { 874 wpipe->pipe_state &= ~PIPE_WANTR; 875 cv_broadcast(&wpipe->pipe_cv); 876 } 877 pipeunlock(wpipe); 878 error = cv_wait_sig(&wpipe->pipe_cv, 879 &wpipe->pipe_lock); 880 881 (void)pipelock(wpipe, 0); 882 if (wpipe->pipe_state & PIPE_EOF) 883 error = EPIPE; 884 } 885 if (error) 886 break; 887 888 /* 889 * If the transfer is large, we can gain performance if 890 * we do process-to-process copies directly. 891 * If the write is non-blocking, we don't use the 892 * direct write mechanism. 893 * 894 * The direct write mechanism will detect the reader going 895 * away on us. 896 */ 897 if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) && 898 (fp->f_flag & FNONBLOCK) == 0 && 899 (wpipe->pipe_map.kva || (amountpipekva < limitpipekva))) { 900 error = pipe_direct_write(fp, wpipe, uio); 901 902 /* 903 * Break out if error occurred, unless it's ENOMEM. 904 * ENOMEM means we failed to allocate some resources 905 * for direct write, so we just fallback to ordinary 906 * write. If the direct write was successful, 907 * process rest of data via ordinary write. 908 */ 909 if (error == 0) 910 continue; 911 912 if (error != ENOMEM) 913 break; 914 } 915 #endif /* PIPE_NODIRECT */ 916 917 space = bp->size - bp->cnt; 918 919 /* Writes of size <= PIPE_BUF must be atomic. */ 920 if ((space < uio->uio_resid) && (uio->uio_resid <= PIPE_BUF)) 921 space = 0; 922 923 if (space > 0) { 924 int size; /* Transfer size */ 925 int segsize; /* first segment to transfer */ 926 927 /* 928 * Transfer size is minimum of uio transfer 929 * and free space in pipe buffer. 930 */ 931 if (space > uio->uio_resid) 932 size = uio->uio_resid; 933 else 934 size = space; 935 /* 936 * First segment to transfer is minimum of 937 * transfer size and contiguous space in 938 * pipe buffer. If first segment to transfer 939 * is less than the transfer size, we've got 940 * a wraparound in the buffer. 941 */ 942 segsize = bp->size - bp->in; 943 if (segsize > size) 944 segsize = size; 945 946 /* Transfer first segment */ 947 error = uiomove((char *)bp->buffer + bp->in, segsize, 948 uio); 949 950 if (error == 0 && segsize < size) { 951 /* 952 * Transfer remaining part now, to 953 * support atomic writes. Wraparound 954 * happened. 955 */ 956 #ifdef DEBUG 957 if (bp->in + segsize != bp->size) 958 panic("Expected pipe buffer wraparound disappeared"); 959 #endif 960 961 error = uiomove(bp->buffer, 962 size - segsize, uio); 963 } 964 if (error) 965 break; 966 967 bp->in += size; 968 if (bp->in >= bp->size) { 969 #ifdef DEBUG 970 if (bp->in != size - segsize + bp->size) 971 panic("Expected wraparound bad"); 972 #endif 973 bp->in = size - segsize; 974 } 975 976 bp->cnt += size; 977 #ifdef DEBUG 978 if (bp->cnt > bp->size) 979 panic("Pipe buffer overflow"); 980 #endif 981 } else { 982 /* 983 * If the "read-side" has been blocked, wake it up now. 984 */ 985 mutex_enter(&wpipe->pipe_lock); 986 if (wpipe->pipe_state & PIPE_WANTR) { 987 wpipe->pipe_state &= ~PIPE_WANTR; 988 cv_broadcast(&wpipe->pipe_cv); 989 } 990 mutex_exit(&wpipe->pipe_lock); 991 992 /* 993 * don't block on non-blocking I/O 994 */ 995 if (fp->f_flag & FNONBLOCK) { 996 error = EAGAIN; 997 break; 998 } 999 1000 /* 1001 * We have no more space and have something to offer, 1002 * wake up select/poll. 1003 */ 1004 if (bp->cnt) 1005 pipeselwakeup(wpipe, wpipe, POLL_OUT); 1006 1007 mutex_enter(&wpipe->pipe_lock); 1008 pipeunlock(wpipe); 1009 wpipe->pipe_state |= PIPE_WANTW; 1010 error = cv_wait_sig(&wpipe->pipe_cv, 1011 &wpipe->pipe_lock); 1012 (void)pipelock(wpipe, 0); 1013 if (error != 0) 1014 break; 1015 /* 1016 * If read side wants to go away, we just issue a signal 1017 * to ourselves. 1018 */ 1019 if (wpipe->pipe_state & PIPE_EOF) { 1020 error = EPIPE; 1021 break; 1022 } 1023 } 1024 } 1025 1026 mutex_enter(&wpipe->pipe_lock); 1027 --wpipe->pipe_busy; 1028 if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANTCLOSE)) { 1029 wpipe->pipe_state &= ~(PIPE_WANTCLOSE | PIPE_WANTR); 1030 cv_broadcast(&wpipe->pipe_cv); 1031 } else if (bp->cnt > 0) { 1032 /* 1033 * If we have put any characters in the buffer, we wake up 1034 * the reader. 1035 */ 1036 if (wpipe->pipe_state & PIPE_WANTR) { 1037 wpipe->pipe_state &= ~PIPE_WANTR; 1038 cv_broadcast(&wpipe->pipe_cv); 1039 } 1040 } 1041 1042 /* 1043 * Don't return EPIPE if I/O was successful 1044 */ 1045 if (error == EPIPE && bp->cnt == 0 && uio->uio_resid == 0) 1046 error = 0; 1047 1048 if (error == 0) 1049 getmicrotime(&wpipe->pipe_mtime); 1050 1051 /* 1052 * We have something to offer, wake up select/poll. 1053 * wpipe->pipe_map.cnt is always 0 in this point (direct write 1054 * is only done synchronously), so check only wpipe->pipe_buffer.cnt 1055 */ 1056 if (bp->cnt) 1057 pipeselwakeup(wpipe, wpipe, POLL_OUT); 1058 1059 /* 1060 * Arrange for next read(2) to do a signal. 1061 */ 1062 wpipe->pipe_state |= PIPE_SIGNALR; 1063 1064 pipeunlock(wpipe); 1065 mutex_exit(&wpipe->pipe_lock); 1066 return (error); 1067 } 1068 1069 /* 1070 * we implement a very minimal set of ioctls for compatibility with sockets. 1071 */ 1072 int 1073 pipe_ioctl(struct file *fp, u_long cmd, void *data, struct lwp *l) 1074 { 1075 struct pipe *pipe = (struct pipe *)fp->f_data; 1076 struct proc *p = l->l_proc; 1077 1078 switch (cmd) { 1079 1080 case FIONBIO: 1081 return (0); 1082 1083 case FIOASYNC: 1084 mutex_enter(&pipe->pipe_lock); 1085 if (*(int *)data) { 1086 pipe->pipe_state |= PIPE_ASYNC; 1087 } else { 1088 pipe->pipe_state &= ~PIPE_ASYNC; 1089 } 1090 mutex_exit(&pipe->pipe_lock); 1091 return (0); 1092 1093 case FIONREAD: 1094 mutex_enter(&pipe->pipe_lock); 1095 #ifndef PIPE_NODIRECT 1096 if (pipe->pipe_state & PIPE_DIRECTW) 1097 *(int *)data = pipe->pipe_map.cnt; 1098 else 1099 #endif 1100 *(int *)data = pipe->pipe_buffer.cnt; 1101 mutex_exit(&pipe->pipe_lock); 1102 return (0); 1103 1104 case FIONWRITE: 1105 /* Look at other side */ 1106 rw_enter(&pipe_peer_lock, RW_READER); 1107 pipe = pipe->pipe_peer; 1108 mutex_enter(&pipe->pipe_lock); 1109 #ifndef PIPE_NODIRECT 1110 if (pipe->pipe_state & PIPE_DIRECTW) 1111 *(int *)data = pipe->pipe_map.cnt; 1112 else 1113 #endif 1114 *(int *)data = pipe->pipe_buffer.cnt; 1115 mutex_exit(&pipe->pipe_lock); 1116 rw_exit(&pipe_peer_lock); 1117 return (0); 1118 1119 case FIONSPACE: 1120 /* Look at other side */ 1121 rw_enter(&pipe_peer_lock, RW_READER); 1122 pipe = pipe->pipe_peer; 1123 mutex_enter(&pipe->pipe_lock); 1124 #ifndef PIPE_NODIRECT 1125 /* 1126 * If we're in direct-mode, we don't really have a 1127 * send queue, and any other write will block. Thus 1128 * zero seems like the best answer. 1129 */ 1130 if (pipe->pipe_state & PIPE_DIRECTW) 1131 *(int *)data = 0; 1132 else 1133 #endif 1134 *(int *)data = pipe->pipe_buffer.size - 1135 pipe->pipe_buffer.cnt; 1136 mutex_exit(&pipe->pipe_lock); 1137 rw_exit(&pipe_peer_lock); 1138 return (0); 1139 1140 case TIOCSPGRP: 1141 case FIOSETOWN: 1142 return fsetown(p, &pipe->pipe_pgid, cmd, data); 1143 1144 case TIOCGPGRP: 1145 case FIOGETOWN: 1146 return fgetown(p, pipe->pipe_pgid, cmd, data); 1147 1148 } 1149 return (EPASSTHROUGH); 1150 } 1151 1152 int 1153 pipe_poll(struct file *fp, int events, struct lwp *l) 1154 { 1155 struct pipe *rpipe = (struct pipe *)fp->f_data; 1156 struct pipe *wpipe; 1157 int eof = 0; 1158 int revents = 0; 1159 1160 retry: 1161 mutex_enter(&rpipe->pipe_lock); 1162 wpipe = rpipe->pipe_peer; 1163 if (wpipe != NULL && mutex_tryenter(&wpipe->pipe_lock) == 0) { 1164 /* Deal with race for peer */ 1165 mutex_exit(&rpipe->pipe_lock); 1166 /* XXX Might be about to deadlock w/kernel_lock. */ 1167 yield(); 1168 goto retry; 1169 } 1170 1171 if (events & (POLLIN | POLLRDNORM)) 1172 if ((rpipe->pipe_buffer.cnt > 0) || 1173 #ifndef PIPE_NODIRECT 1174 (rpipe->pipe_state & PIPE_DIRECTR) || 1175 #endif 1176 (rpipe->pipe_state & PIPE_EOF)) 1177 revents |= events & (POLLIN | POLLRDNORM); 1178 1179 eof |= (rpipe->pipe_state & PIPE_EOF); 1180 mutex_exit(&rpipe->pipe_lock); 1181 1182 if (wpipe == NULL) 1183 revents |= events & (POLLOUT | POLLWRNORM); 1184 else { 1185 if (events & (POLLOUT | POLLWRNORM)) 1186 if ((wpipe->pipe_state & PIPE_EOF) || ( 1187 #ifndef PIPE_NODIRECT 1188 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 1189 #endif 1190 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 1191 revents |= events & (POLLOUT | POLLWRNORM); 1192 1193 eof |= (wpipe->pipe_state & PIPE_EOF); 1194 mutex_exit(&wpipe->pipe_lock); 1195 } 1196 1197 if (wpipe == NULL || eof) 1198 revents |= POLLHUP; 1199 1200 if (revents == 0) { 1201 if (events & (POLLIN | POLLRDNORM)) 1202 selrecord(l, &rpipe->pipe_sel); 1203 1204 if (events & (POLLOUT | POLLWRNORM)) 1205 selrecord(l, &wpipe->pipe_sel); 1206 } 1207 1208 return (revents); 1209 } 1210 1211 static int 1212 pipe_stat(struct file *fp, struct stat *ub, struct lwp *l) 1213 { 1214 struct pipe *pipe = (struct pipe *)fp->f_data; 1215 1216 rw_enter(&pipe_peer_lock, RW_READER); 1217 1218 memset((void *)ub, 0, sizeof(*ub)); 1219 ub->st_mode = S_IFIFO | S_IRUSR | S_IWUSR; 1220 ub->st_blksize = pipe->pipe_buffer.size; 1221 if (ub->st_blksize == 0 && pipe->pipe_peer) 1222 ub->st_blksize = pipe->pipe_peer->pipe_buffer.size; 1223 ub->st_size = pipe->pipe_buffer.cnt; 1224 ub->st_blocks = (ub->st_size) ? 1 : 0; 1225 TIMEVAL_TO_TIMESPEC(&pipe->pipe_atime, &ub->st_atimespec); 1226 TIMEVAL_TO_TIMESPEC(&pipe->pipe_mtime, &ub->st_mtimespec); 1227 TIMEVAL_TO_TIMESPEC(&pipe->pipe_ctime, &ub->st_ctimespec); 1228 ub->st_uid = kauth_cred_geteuid(fp->f_cred); 1229 ub->st_gid = kauth_cred_getegid(fp->f_cred); 1230 1231 rw_exit(&pipe_peer_lock); 1232 1233 /* 1234 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 1235 * XXX (st_dev, st_ino) should be unique. 1236 */ 1237 return (0); 1238 } 1239 1240 /* ARGSUSED */ 1241 static int 1242 pipe_close(struct file *fp, struct lwp *l) 1243 { 1244 struct pipe *pipe = (struct pipe *)fp->f_data; 1245 1246 fp->f_data = NULL; 1247 pipeclose(fp, pipe); 1248 return (0); 1249 } 1250 1251 static void 1252 pipe_free_kmem(struct pipe *pipe) 1253 { 1254 1255 if (pipe->pipe_buffer.buffer != NULL) { 1256 if (pipe->pipe_buffer.size > PIPE_SIZE) 1257 --nbigpipe; 1258 amountpipekva -= pipe->pipe_buffer.size; 1259 uvm_km_free(kernel_map, 1260 (vaddr_t)pipe->pipe_buffer.buffer, 1261 pipe->pipe_buffer.size, UVM_KMF_PAGEABLE); 1262 pipe->pipe_buffer.buffer = NULL; 1263 } 1264 #ifndef PIPE_NODIRECT 1265 if (pipe->pipe_map.kva != 0) { 1266 pipe_loan_free(pipe); 1267 pipe->pipe_map.cnt = 0; 1268 pipe->pipe_map.kva = 0; 1269 pipe->pipe_map.pos = 0; 1270 pipe->pipe_map.npages = 0; 1271 } 1272 #endif /* !PIPE_NODIRECT */ 1273 } 1274 1275 /* 1276 * shutdown the pipe 1277 */ 1278 static void 1279 pipeclose(struct file *fp, struct pipe *pipe) 1280 { 1281 struct pipe *ppipe; 1282 1283 if (pipe == NULL) 1284 return; 1285 1286 retry: 1287 rw_enter(&pipe_peer_lock, RW_WRITER); 1288 mutex_enter(&pipe->pipe_lock); 1289 1290 pipeselwakeup(pipe, pipe, POLL_HUP); 1291 1292 /* 1293 * If the other side is blocked, wake it up saying that 1294 * we want to close it down. 1295 */ 1296 pipe->pipe_state |= PIPE_EOF; 1297 if (pipe->pipe_busy) { 1298 rw_exit(&pipe_peer_lock); 1299 while (pipe->pipe_busy) { 1300 cv_broadcast(&pipe->pipe_cv); 1301 pipe->pipe_state |= PIPE_WANTCLOSE; 1302 cv_wait_sig(&pipe->pipe_cv, &pipe->pipe_lock); 1303 } 1304 if (!rw_tryenter(&pipe_peer_lock, RW_READER)) { 1305 mutex_exit(&pipe->pipe_lock); 1306 /* XXX Might be about to deadlock w/kernel_lock. */ 1307 yield(); 1308 goto retry; 1309 } 1310 } 1311 1312 /* 1313 * Disconnect from peer 1314 */ 1315 if ((ppipe = pipe->pipe_peer) != NULL) { 1316 /* Deal with race for peer */ 1317 if (mutex_tryenter(&ppipe->pipe_lock) == 0) { 1318 mutex_exit(&pipe->pipe_lock); 1319 rw_exit(&pipe_peer_lock); 1320 /* XXX Might be about to deadlock w/kernel_lock. */ 1321 yield(); 1322 goto retry; 1323 } 1324 pipeselwakeup(ppipe, ppipe, POLL_HUP); 1325 1326 ppipe->pipe_state |= PIPE_EOF; 1327 cv_broadcast(&ppipe->pipe_cv); 1328 ppipe->pipe_peer = NULL; 1329 mutex_exit(&ppipe->pipe_lock); 1330 } 1331 1332 KASSERT((pipe->pipe_state & PIPE_LOCKFL) == 0); 1333 1334 mutex_exit(&pipe->pipe_lock); 1335 rw_exit(&pipe_peer_lock); 1336 1337 /* 1338 * free resources 1339 */ 1340 pipe_free_kmem(pipe); 1341 mutex_destroy(&pipe->pipe_lock); 1342 cv_destroy(&pipe->pipe_cv); 1343 cv_destroy(&pipe->pipe_lkcv); 1344 pool_put(&pipe_pool, pipe); 1345 } 1346 1347 static void 1348 filt_pipedetach(struct knote *kn) 1349 { 1350 struct pipe *pipe = (struct pipe *)kn->kn_fp->f_data; 1351 1352 rw_enter(&pipe_peer_lock, RW_READER); 1353 1354 switch(kn->kn_filter) { 1355 case EVFILT_WRITE: 1356 /* need the peer structure, not our own */ 1357 pipe = pipe->pipe_peer; 1358 1359 /* if reader end already closed, just return */ 1360 if (pipe == NULL) { 1361 rw_exit(&pipe_peer_lock); 1362 return; 1363 } 1364 1365 break; 1366 default: 1367 /* nothing to do */ 1368 break; 1369 } 1370 1371 #ifdef DIAGNOSTIC 1372 if (kn->kn_hook != pipe) 1373 panic("filt_pipedetach: inconsistent knote"); 1374 #endif 1375 1376 mutex_enter(&pipe->pipe_lock); 1377 SLIST_REMOVE(&pipe->pipe_sel.sel_klist, kn, knote, kn_selnext); 1378 mutex_exit(&pipe->pipe_lock); 1379 rw_exit(&pipe_peer_lock); 1380 } 1381 1382 /*ARGSUSED*/ 1383 static int 1384 filt_piperead(struct knote *kn, long hint) 1385 { 1386 struct pipe *rpipe = (struct pipe *)kn->kn_fp->f_data; 1387 struct pipe *wpipe; 1388 1389 if ((hint & NOTE_SUBMIT) == 0) { 1390 rw_enter(&pipe_peer_lock, RW_READER); 1391 mutex_enter(&rpipe->pipe_lock); 1392 } 1393 wpipe = rpipe->pipe_peer; 1394 kn->kn_data = rpipe->pipe_buffer.cnt; 1395 1396 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1397 kn->kn_data = rpipe->pipe_map.cnt; 1398 1399 if ((rpipe->pipe_state & PIPE_EOF) || 1400 (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1401 kn->kn_flags |= EV_EOF; 1402 if ((hint & NOTE_SUBMIT) == 0) { 1403 mutex_exit(&rpipe->pipe_lock); 1404 rw_exit(&pipe_peer_lock); 1405 } 1406 return (1); 1407 } 1408 1409 if ((hint & NOTE_SUBMIT) == 0) { 1410 mutex_exit(&rpipe->pipe_lock); 1411 rw_exit(&pipe_peer_lock); 1412 } 1413 return (kn->kn_data > 0); 1414 } 1415 1416 /*ARGSUSED*/ 1417 static int 1418 filt_pipewrite(struct knote *kn, long hint) 1419 { 1420 struct pipe *rpipe = (struct pipe *)kn->kn_fp->f_data; 1421 struct pipe *wpipe; 1422 1423 if ((hint & NOTE_SUBMIT) == 0) { 1424 rw_enter(&pipe_peer_lock, RW_READER); 1425 mutex_enter(&rpipe->pipe_lock); 1426 } 1427 wpipe = rpipe->pipe_peer; 1428 1429 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1430 kn->kn_data = 0; 1431 kn->kn_flags |= EV_EOF; 1432 if ((hint & NOTE_SUBMIT) == 0) { 1433 mutex_exit(&rpipe->pipe_lock); 1434 rw_exit(&pipe_peer_lock); 1435 } 1436 return (1); 1437 } 1438 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1439 if (wpipe->pipe_state & PIPE_DIRECTW) 1440 kn->kn_data = 0; 1441 1442 if ((hint & NOTE_SUBMIT) == 0) { 1443 mutex_exit(&rpipe->pipe_lock); 1444 rw_exit(&pipe_peer_lock); 1445 } 1446 return (kn->kn_data >= PIPE_BUF); 1447 } 1448 1449 static const struct filterops pipe_rfiltops = 1450 { 1, NULL, filt_pipedetach, filt_piperead }; 1451 static const struct filterops pipe_wfiltops = 1452 { 1, NULL, filt_pipedetach, filt_pipewrite }; 1453 1454 /*ARGSUSED*/ 1455 static int 1456 pipe_kqfilter(struct file *fp, struct knote *kn) 1457 { 1458 struct pipe *pipe; 1459 1460 rw_enter(&pipe_peer_lock, RW_READER); 1461 pipe = (struct pipe *)kn->kn_fp->f_data; 1462 1463 switch (kn->kn_filter) { 1464 case EVFILT_READ: 1465 kn->kn_fop = &pipe_rfiltops; 1466 break; 1467 case EVFILT_WRITE: 1468 kn->kn_fop = &pipe_wfiltops; 1469 pipe = pipe->pipe_peer; 1470 if (pipe == NULL) { 1471 /* other end of pipe has been closed */ 1472 rw_exit(&pipe_peer_lock); 1473 return (EBADF); 1474 } 1475 break; 1476 default: 1477 rw_exit(&pipe_peer_lock); 1478 return (1); 1479 } 1480 1481 kn->kn_hook = pipe; 1482 mutex_enter(&pipe->pipe_lock); 1483 SLIST_INSERT_HEAD(&pipe->pipe_sel.sel_klist, kn, kn_selnext); 1484 mutex_exit(&pipe->pipe_lock); 1485 rw_exit(&pipe_peer_lock); 1486 1487 return (0); 1488 } 1489 1490 /* 1491 * Handle pipe sysctls. 1492 */ 1493 SYSCTL_SETUP(sysctl_kern_pipe_setup, "sysctl kern.pipe subtree setup") 1494 { 1495 1496 sysctl_createv(clog, 0, NULL, NULL, 1497 CTLFLAG_PERMANENT, 1498 CTLTYPE_NODE, "kern", NULL, 1499 NULL, 0, NULL, 0, 1500 CTL_KERN, CTL_EOL); 1501 sysctl_createv(clog, 0, NULL, NULL, 1502 CTLFLAG_PERMANENT, 1503 CTLTYPE_NODE, "pipe", 1504 SYSCTL_DESCR("Pipe settings"), 1505 NULL, 0, NULL, 0, 1506 CTL_KERN, KERN_PIPE, CTL_EOL); 1507 1508 sysctl_createv(clog, 0, NULL, NULL, 1509 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1510 CTLTYPE_INT, "maxkvasz", 1511 SYSCTL_DESCR("Maximum amount of kernel memory to be " 1512 "used for pipes"), 1513 NULL, 0, &maxpipekva, 0, 1514 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXKVASZ, CTL_EOL); 1515 sysctl_createv(clog, 0, NULL, NULL, 1516 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1517 CTLTYPE_INT, "maxloankvasz", 1518 SYSCTL_DESCR("Limit for direct transfers via page loan"), 1519 NULL, 0, &limitpipekva, 0, 1520 CTL_KERN, KERN_PIPE, KERN_PIPE_LIMITKVA, CTL_EOL); 1521 sysctl_createv(clog, 0, NULL, NULL, 1522 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1523 CTLTYPE_INT, "maxbigpipes", 1524 SYSCTL_DESCR("Maximum number of \"big\" pipes"), 1525 NULL, 0, &maxbigpipes, 0, 1526 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXBIGPIPES, CTL_EOL); 1527 sysctl_createv(clog, 0, NULL, NULL, 1528 CTLFLAG_PERMANENT, 1529 CTLTYPE_INT, "nbigpipes", 1530 SYSCTL_DESCR("Number of \"big\" pipes"), 1531 NULL, 0, &nbigpipe, 0, 1532 CTL_KERN, KERN_PIPE, KERN_PIPE_NBIGPIPES, CTL_EOL); 1533 sysctl_createv(clog, 0, NULL, NULL, 1534 CTLFLAG_PERMANENT, 1535 CTLTYPE_INT, "kvasize", 1536 SYSCTL_DESCR("Amount of kernel memory consumed by pipe " 1537 "buffers"), 1538 NULL, 0, &amountpipekva, 0, 1539 CTL_KERN, KERN_PIPE, KERN_PIPE_KVASIZE, CTL_EOL); 1540 } 1541