1 /* $NetBSD: sys_pipe.c,v 1.128 2010/08/11 11:46:32 pgoyette Exp $ */ 2 3 /*- 4 * Copyright (c) 2003, 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Paul Kranenburg, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1996 John S. Dyson 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice immediately at the beginning of the file, without modification, 41 * this list of conditions, and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. Absolutely no warranty of function or purpose is made by the author 46 * John S. Dyson. 47 * 4. Modifications may be freely made to this file if the above conditions 48 * are met. 49 */ 50 51 /* 52 * This file contains a high-performance replacement for the socket-based 53 * pipes scheme originally used. It does not support all features of 54 * sockets, but does do everything that pipes normally do. 55 * 56 * This code has two modes of operation, a small write mode and a large 57 * write mode. The small write mode acts like conventional pipes with 58 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 59 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 60 * and PIPE_SIZE in size it is mapped read-only into the kernel address space 61 * using the UVM page loan facility from where the receiving process can copy 62 * the data directly from the pages in the sending process. 63 * 64 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 65 * happen for small transfers so that the system will not spend all of 66 * its time context switching. PIPE_SIZE is constrained by the 67 * amount of kernel virtual memory. 68 */ 69 70 #include <sys/cdefs.h> 71 __KERNEL_RCSID(0, "$NetBSD: sys_pipe.c,v 1.128 2010/08/11 11:46:32 pgoyette Exp $"); 72 73 #include <sys/param.h> 74 #include <sys/systm.h> 75 #include <sys/proc.h> 76 #include <sys/fcntl.h> 77 #include <sys/file.h> 78 #include <sys/filedesc.h> 79 #include <sys/filio.h> 80 #include <sys/kernel.h> 81 #include <sys/ttycom.h> 82 #include <sys/stat.h> 83 #include <sys/poll.h> 84 #include <sys/signalvar.h> 85 #include <sys/vnode.h> 86 #include <sys/uio.h> 87 #include <sys/select.h> 88 #include <sys/mount.h> 89 #include <sys/syscallargs.h> 90 #include <sys/sysctl.h> 91 #include <sys/kauth.h> 92 #include <sys/atomic.h> 93 #include <sys/pipe.h> 94 95 #include <uvm/uvm.h> 96 97 /* 98 * Use this to disable direct I/O and decrease the code size: 99 * #define PIPE_NODIRECT 100 */ 101 102 /* XXX Disabled for now; rare hangs switching between direct/buffered */ 103 #define PIPE_NODIRECT 104 105 static int pipe_read(file_t *, off_t *, struct uio *, kauth_cred_t, int); 106 static int pipe_write(file_t *, off_t *, struct uio *, kauth_cred_t, int); 107 static int pipe_close(file_t *); 108 static int pipe_poll(file_t *, int); 109 static int pipe_kqfilter(file_t *, struct knote *); 110 static int pipe_stat(file_t *, struct stat *); 111 static int pipe_ioctl(file_t *, u_long, void *); 112 static void pipe_restart(file_t *); 113 114 static const struct fileops pipeops = { 115 .fo_read = pipe_read, 116 .fo_write = pipe_write, 117 .fo_ioctl = pipe_ioctl, 118 .fo_fcntl = fnullop_fcntl, 119 .fo_poll = pipe_poll, 120 .fo_stat = pipe_stat, 121 .fo_close = pipe_close, 122 .fo_kqfilter = pipe_kqfilter, 123 .fo_restart = pipe_restart, 124 }; 125 126 /* 127 * Default pipe buffer size(s), this can be kind-of large now because pipe 128 * space is pageable. The pipe code will try to maintain locality of 129 * reference for performance reasons, so small amounts of outstanding I/O 130 * will not wipe the cache. 131 */ 132 #define MINPIPESIZE (PIPE_SIZE / 3) 133 #define MAXPIPESIZE (2 * PIPE_SIZE / 3) 134 135 /* 136 * Maximum amount of kva for pipes -- this is kind-of a soft limit, but 137 * is there so that on large systems, we don't exhaust it. 138 */ 139 #define MAXPIPEKVA (8 * 1024 * 1024) 140 static u_int maxpipekva = MAXPIPEKVA; 141 142 /* 143 * Limit for direct transfers, we cannot, of course limit 144 * the amount of kva for pipes in general though. 145 */ 146 #define LIMITPIPEKVA (16 * 1024 * 1024) 147 static u_int limitpipekva = LIMITPIPEKVA; 148 149 /* 150 * Limit the number of "big" pipes 151 */ 152 #define LIMITBIGPIPES 32 153 static u_int maxbigpipes = LIMITBIGPIPES; 154 static u_int nbigpipe = 0; 155 156 /* 157 * Amount of KVA consumed by pipe buffers. 158 */ 159 static u_int amountpipekva = 0; 160 161 static void pipeclose(struct pipe *); 162 static void pipe_free_kmem(struct pipe *); 163 static int pipe_create(struct pipe **, pool_cache_t); 164 static int pipelock(struct pipe *, int); 165 static inline void pipeunlock(struct pipe *); 166 static void pipeselwakeup(struct pipe *, struct pipe *, int); 167 #ifndef PIPE_NODIRECT 168 static int pipe_direct_write(file_t *, struct pipe *, struct uio *); 169 #endif 170 static int pipespace(struct pipe *, int); 171 static int pipe_ctor(void *, void *, int); 172 static void pipe_dtor(void *, void *); 173 174 #ifndef PIPE_NODIRECT 175 static int pipe_loan_alloc(struct pipe *, int); 176 static void pipe_loan_free(struct pipe *); 177 #endif /* PIPE_NODIRECT */ 178 179 static pool_cache_t pipe_wr_cache; 180 static pool_cache_t pipe_rd_cache; 181 182 void 183 pipe_init(void) 184 { 185 186 /* Writer side is not automatically allocated KVA. */ 187 pipe_wr_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "pipewr", 188 NULL, IPL_NONE, pipe_ctor, pipe_dtor, NULL); 189 KASSERT(pipe_wr_cache != NULL); 190 191 /* Reader side gets preallocated KVA. */ 192 pipe_rd_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "piperd", 193 NULL, IPL_NONE, pipe_ctor, pipe_dtor, (void *)1); 194 KASSERT(pipe_rd_cache != NULL); 195 } 196 197 static int 198 pipe_ctor(void *arg, void *obj, int flags) 199 { 200 struct pipe *pipe; 201 vaddr_t va; 202 203 pipe = obj; 204 205 memset(pipe, 0, sizeof(struct pipe)); 206 if (arg != NULL) { 207 /* Preallocate space. */ 208 va = uvm_km_alloc(kernel_map, PIPE_SIZE, 0, 209 UVM_KMF_PAGEABLE | UVM_KMF_WAITVA); 210 KASSERT(va != 0); 211 pipe->pipe_kmem = va; 212 atomic_add_int(&amountpipekva, PIPE_SIZE); 213 } 214 cv_init(&pipe->pipe_rcv, "pipe_rd"); 215 cv_init(&pipe->pipe_wcv, "pipe_wr"); 216 cv_init(&pipe->pipe_draincv, "pipe_drn"); 217 cv_init(&pipe->pipe_lkcv, "pipe_lk"); 218 selinit(&pipe->pipe_sel); 219 pipe->pipe_state = PIPE_SIGNALR; 220 221 return 0; 222 } 223 224 static void 225 pipe_dtor(void *arg, void *obj) 226 { 227 struct pipe *pipe; 228 229 pipe = obj; 230 231 cv_destroy(&pipe->pipe_rcv); 232 cv_destroy(&pipe->pipe_wcv); 233 cv_destroy(&pipe->pipe_draincv); 234 cv_destroy(&pipe->pipe_lkcv); 235 seldestroy(&pipe->pipe_sel); 236 if (pipe->pipe_kmem != 0) { 237 uvm_km_free(kernel_map, pipe->pipe_kmem, PIPE_SIZE, 238 UVM_KMF_PAGEABLE); 239 atomic_add_int(&amountpipekva, -PIPE_SIZE); 240 } 241 } 242 243 /* 244 * The pipe system call for the DTYPE_PIPE type of pipes 245 */ 246 int 247 sys_pipe(struct lwp *l, const void *v, register_t *retval) 248 { 249 struct pipe *rpipe, *wpipe; 250 file_t *rf, *wf; 251 int fd, error; 252 proc_t *p; 253 254 p = curproc; 255 rpipe = wpipe = NULL; 256 if (pipe_create(&rpipe, pipe_rd_cache) || 257 pipe_create(&wpipe, pipe_wr_cache)) { 258 error = ENOMEM; 259 goto free2; 260 } 261 rpipe->pipe_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 262 wpipe->pipe_lock = rpipe->pipe_lock; 263 mutex_obj_hold(wpipe->pipe_lock); 264 265 error = fd_allocfile(&rf, &fd); 266 if (error) 267 goto free2; 268 retval[0] = fd; 269 rf->f_flag = FREAD; 270 rf->f_type = DTYPE_PIPE; 271 rf->f_data = (void *)rpipe; 272 rf->f_ops = &pipeops; 273 274 error = fd_allocfile(&wf, &fd); 275 if (error) 276 goto free3; 277 retval[1] = fd; 278 wf->f_flag = FWRITE; 279 wf->f_type = DTYPE_PIPE; 280 wf->f_data = (void *)wpipe; 281 wf->f_ops = &pipeops; 282 283 rpipe->pipe_peer = wpipe; 284 wpipe->pipe_peer = rpipe; 285 286 fd_affix(p, rf, (int)retval[0]); 287 fd_affix(p, wf, (int)retval[1]); 288 return (0); 289 free3: 290 fd_abort(p, rf, (int)retval[0]); 291 free2: 292 pipeclose(wpipe); 293 pipeclose(rpipe); 294 295 return (error); 296 } 297 298 /* 299 * Allocate kva for pipe circular buffer, the space is pageable 300 * This routine will 'realloc' the size of a pipe safely, if it fails 301 * it will retain the old buffer. 302 * If it fails it will return ENOMEM. 303 */ 304 static int 305 pipespace(struct pipe *pipe, int size) 306 { 307 void *buffer; 308 309 /* 310 * Allocate pageable virtual address space. Physical memory is 311 * allocated on demand. 312 */ 313 if (size == PIPE_SIZE && pipe->pipe_kmem != 0) { 314 buffer = (void *)pipe->pipe_kmem; 315 } else { 316 buffer = (void *)uvm_km_alloc(kernel_map, round_page(size), 317 0, UVM_KMF_PAGEABLE); 318 if (buffer == NULL) 319 return (ENOMEM); 320 atomic_add_int(&amountpipekva, size); 321 } 322 323 /* free old resources if we're resizing */ 324 pipe_free_kmem(pipe); 325 pipe->pipe_buffer.buffer = buffer; 326 pipe->pipe_buffer.size = size; 327 pipe->pipe_buffer.in = 0; 328 pipe->pipe_buffer.out = 0; 329 pipe->pipe_buffer.cnt = 0; 330 return (0); 331 } 332 333 /* 334 * Initialize and allocate VM and memory for pipe. 335 */ 336 static int 337 pipe_create(struct pipe **pipep, pool_cache_t cache) 338 { 339 struct pipe *pipe; 340 int error; 341 342 pipe = pool_cache_get(cache, PR_WAITOK); 343 KASSERT(pipe != NULL); 344 *pipep = pipe; 345 error = 0; 346 getnanotime(&pipe->pipe_btime); 347 pipe->pipe_atime = pipe->pipe_mtime = pipe->pipe_btime; 348 pipe->pipe_lock = NULL; 349 if (cache == pipe_rd_cache) { 350 error = pipespace(pipe, PIPE_SIZE); 351 } else { 352 pipe->pipe_buffer.buffer = NULL; 353 pipe->pipe_buffer.size = 0; 354 pipe->pipe_buffer.in = 0; 355 pipe->pipe_buffer.out = 0; 356 pipe->pipe_buffer.cnt = 0; 357 } 358 return error; 359 } 360 361 /* 362 * Lock a pipe for I/O, blocking other access 363 * Called with pipe spin lock held. 364 */ 365 static int 366 pipelock(struct pipe *pipe, int catch) 367 { 368 int error; 369 370 KASSERT(mutex_owned(pipe->pipe_lock)); 371 372 while (pipe->pipe_state & PIPE_LOCKFL) { 373 pipe->pipe_state |= PIPE_LWANT; 374 if (catch) { 375 error = cv_wait_sig(&pipe->pipe_lkcv, pipe->pipe_lock); 376 if (error != 0) 377 return error; 378 } else 379 cv_wait(&pipe->pipe_lkcv, pipe->pipe_lock); 380 } 381 382 pipe->pipe_state |= PIPE_LOCKFL; 383 384 return 0; 385 } 386 387 /* 388 * unlock a pipe I/O lock 389 */ 390 static inline void 391 pipeunlock(struct pipe *pipe) 392 { 393 394 KASSERT(pipe->pipe_state & PIPE_LOCKFL); 395 396 pipe->pipe_state &= ~PIPE_LOCKFL; 397 if (pipe->pipe_state & PIPE_LWANT) { 398 pipe->pipe_state &= ~PIPE_LWANT; 399 cv_broadcast(&pipe->pipe_lkcv); 400 } 401 } 402 403 /* 404 * Select/poll wakup. This also sends SIGIO to peer connected to 405 * 'sigpipe' side of pipe. 406 */ 407 static void 408 pipeselwakeup(struct pipe *selp, struct pipe *sigp, int code) 409 { 410 int band; 411 412 switch (code) { 413 case POLL_IN: 414 band = POLLIN|POLLRDNORM; 415 break; 416 case POLL_OUT: 417 band = POLLOUT|POLLWRNORM; 418 break; 419 case POLL_HUP: 420 band = POLLHUP; 421 break; 422 case POLL_ERR: 423 band = POLLERR; 424 break; 425 default: 426 band = 0; 427 #ifdef DIAGNOSTIC 428 printf("bad siginfo code %d in pipe notification.\n", code); 429 #endif 430 break; 431 } 432 433 selnotify(&selp->pipe_sel, band, NOTE_SUBMIT); 434 435 if (sigp == NULL || (sigp->pipe_state & PIPE_ASYNC) == 0) 436 return; 437 438 fownsignal(sigp->pipe_pgid, SIGIO, code, band, selp); 439 } 440 441 static int 442 pipe_read(file_t *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 443 int flags) 444 { 445 struct pipe *rpipe = (struct pipe *) fp->f_data; 446 struct pipebuf *bp = &rpipe->pipe_buffer; 447 kmutex_t *lock = rpipe->pipe_lock; 448 int error; 449 size_t nread = 0; 450 size_t size; 451 size_t ocnt; 452 unsigned int wakeup_state = 0; 453 454 mutex_enter(lock); 455 ++rpipe->pipe_busy; 456 ocnt = bp->cnt; 457 458 again: 459 error = pipelock(rpipe, 1); 460 if (error) 461 goto unlocked_error; 462 463 while (uio->uio_resid) { 464 /* 465 * Normal pipe buffer receive. 466 */ 467 if (bp->cnt > 0) { 468 size = bp->size - bp->out; 469 if (size > bp->cnt) 470 size = bp->cnt; 471 if (size > uio->uio_resid) 472 size = uio->uio_resid; 473 474 mutex_exit(lock); 475 error = uiomove((char *)bp->buffer + bp->out, size, uio); 476 mutex_enter(lock); 477 if (error) 478 break; 479 480 bp->out += size; 481 if (bp->out >= bp->size) 482 bp->out = 0; 483 484 bp->cnt -= size; 485 486 /* 487 * If there is no more to read in the pipe, reset 488 * its pointers to the beginning. This improves 489 * cache hit stats. 490 */ 491 if (bp->cnt == 0) { 492 bp->in = 0; 493 bp->out = 0; 494 } 495 nread += size; 496 continue; 497 } 498 499 #ifndef PIPE_NODIRECT 500 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) { 501 /* 502 * Direct copy, bypassing a kernel buffer. 503 */ 504 void *va; 505 u_int gen; 506 507 KASSERT(rpipe->pipe_state & PIPE_DIRECTW); 508 509 size = rpipe->pipe_map.cnt; 510 if (size > uio->uio_resid) 511 size = uio->uio_resid; 512 513 va = (char *)rpipe->pipe_map.kva + rpipe->pipe_map.pos; 514 gen = rpipe->pipe_map.egen; 515 mutex_exit(lock); 516 517 /* 518 * Consume emap and read the data from loaned pages. 519 */ 520 uvm_emap_consume(gen); 521 error = uiomove(va, size, uio); 522 523 mutex_enter(lock); 524 if (error) 525 break; 526 nread += size; 527 rpipe->pipe_map.pos += size; 528 rpipe->pipe_map.cnt -= size; 529 if (rpipe->pipe_map.cnt == 0) { 530 rpipe->pipe_state &= ~PIPE_DIRECTR; 531 cv_broadcast(&rpipe->pipe_wcv); 532 } 533 continue; 534 } 535 #endif 536 /* 537 * Break if some data was read. 538 */ 539 if (nread > 0) 540 break; 541 542 /* 543 * Detect EOF condition. 544 * Read returns 0 on EOF, no need to set error. 545 */ 546 if (rpipe->pipe_state & PIPE_EOF) 547 break; 548 549 /* 550 * Don't block on non-blocking I/O. 551 */ 552 if (fp->f_flag & FNONBLOCK) { 553 error = EAGAIN; 554 break; 555 } 556 557 /* 558 * Unlock the pipe buffer for our remaining processing. 559 * We will either break out with an error or we will 560 * sleep and relock to loop. 561 */ 562 pipeunlock(rpipe); 563 564 /* 565 * Re-check to see if more direct writes are pending. 566 */ 567 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) 568 goto again; 569 570 #if 1 /* XXX (dsl) I'm sure these aren't needed here ... */ 571 /* 572 * We want to read more, wake up select/poll. 573 */ 574 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 575 576 /* 577 * If the "write-side" is blocked, wake it up now. 578 */ 579 cv_broadcast(&rpipe->pipe_wcv); 580 #endif 581 582 if (wakeup_state & PIPE_RESTART) { 583 error = ERESTART; 584 goto unlocked_error; 585 } 586 587 /* Now wait until the pipe is filled */ 588 error = cv_wait_sig(&rpipe->pipe_rcv, lock); 589 if (error != 0) 590 goto unlocked_error; 591 wakeup_state = rpipe->pipe_state; 592 goto again; 593 } 594 595 if (error == 0) 596 getnanotime(&rpipe->pipe_atime); 597 pipeunlock(rpipe); 598 599 unlocked_error: 600 --rpipe->pipe_busy; 601 if (rpipe->pipe_busy == 0) { 602 rpipe->pipe_state &= ~PIPE_RESTART; 603 cv_broadcast(&rpipe->pipe_draincv); 604 } 605 if (bp->cnt < MINPIPESIZE) { 606 cv_broadcast(&rpipe->pipe_wcv); 607 } 608 609 /* 610 * If anything was read off the buffer, signal to the writer it's 611 * possible to write more data. Also send signal if we are here for the 612 * first time after last write. 613 */ 614 if ((bp->size - bp->cnt) >= PIPE_BUF 615 && (ocnt != bp->cnt || (rpipe->pipe_state & PIPE_SIGNALR))) { 616 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 617 rpipe->pipe_state &= ~PIPE_SIGNALR; 618 } 619 620 mutex_exit(lock); 621 return (error); 622 } 623 624 #ifndef PIPE_NODIRECT 625 /* 626 * Allocate structure for loan transfer. 627 */ 628 static int 629 pipe_loan_alloc(struct pipe *wpipe, int npages) 630 { 631 vsize_t len; 632 633 len = (vsize_t)npages << PAGE_SHIFT; 634 atomic_add_int(&amountpipekva, len); 635 wpipe->pipe_map.kva = uvm_km_alloc(kernel_map, len, 0, 636 UVM_KMF_VAONLY | UVM_KMF_WAITVA); 637 if (wpipe->pipe_map.kva == 0) { 638 atomic_add_int(&amountpipekva, -len); 639 return (ENOMEM); 640 } 641 642 wpipe->pipe_map.npages = npages; 643 wpipe->pipe_map.pgs = kmem_alloc(npages * sizeof(struct vm_page *), 644 KM_SLEEP); 645 return (0); 646 } 647 648 /* 649 * Free resources allocated for loan transfer. 650 */ 651 static void 652 pipe_loan_free(struct pipe *wpipe) 653 { 654 vsize_t len; 655 656 len = (vsize_t)wpipe->pipe_map.npages << PAGE_SHIFT; 657 uvm_emap_remove(wpipe->pipe_map.kva, len); /* XXX */ 658 uvm_km_free(kernel_map, wpipe->pipe_map.kva, len, UVM_KMF_VAONLY); 659 wpipe->pipe_map.kva = 0; 660 atomic_add_int(&amountpipekva, -len); 661 kmem_free(wpipe->pipe_map.pgs, 662 wpipe->pipe_map.npages * sizeof(struct vm_page *)); 663 wpipe->pipe_map.pgs = NULL; 664 } 665 666 /* 667 * NetBSD direct write, using uvm_loan() mechanism. 668 * This implements the pipe buffer write mechanism. Note that only 669 * a direct write OR a normal pipe write can be pending at any given time. 670 * If there are any characters in the pipe buffer, the direct write will 671 * be deferred until the receiving process grabs all of the bytes from 672 * the pipe buffer. Then the direct mapping write is set-up. 673 * 674 * Called with the long-term pipe lock held. 675 */ 676 static int 677 pipe_direct_write(file_t *fp, struct pipe *wpipe, struct uio *uio) 678 { 679 struct vm_page **pgs; 680 vaddr_t bbase, base, bend; 681 vsize_t blen, bcnt; 682 int error, npages; 683 voff_t bpos; 684 kmutex_t *lock = wpipe->pipe_lock; 685 686 KASSERT(mutex_owned(wpipe->pipe_lock)); 687 KASSERT(wpipe->pipe_map.cnt == 0); 688 689 mutex_exit(lock); 690 691 /* 692 * Handle first PIPE_CHUNK_SIZE bytes of buffer. Deal with buffers 693 * not aligned to PAGE_SIZE. 694 */ 695 bbase = (vaddr_t)uio->uio_iov->iov_base; 696 base = trunc_page(bbase); 697 bend = round_page(bbase + uio->uio_iov->iov_len); 698 blen = bend - base; 699 bpos = bbase - base; 700 701 if (blen > PIPE_DIRECT_CHUNK) { 702 blen = PIPE_DIRECT_CHUNK; 703 bend = base + blen; 704 bcnt = PIPE_DIRECT_CHUNK - bpos; 705 } else { 706 bcnt = uio->uio_iov->iov_len; 707 } 708 npages = blen >> PAGE_SHIFT; 709 710 /* 711 * Free the old kva if we need more pages than we have 712 * allocated. 713 */ 714 if (wpipe->pipe_map.kva != 0 && npages > wpipe->pipe_map.npages) 715 pipe_loan_free(wpipe); 716 717 /* Allocate new kva. */ 718 if (wpipe->pipe_map.kva == 0) { 719 error = pipe_loan_alloc(wpipe, npages); 720 if (error) { 721 mutex_enter(lock); 722 return (error); 723 } 724 } 725 726 /* Loan the write buffer memory from writer process */ 727 pgs = wpipe->pipe_map.pgs; 728 error = uvm_loan(&uio->uio_vmspace->vm_map, base, blen, 729 pgs, UVM_LOAN_TOPAGE); 730 if (error) { 731 pipe_loan_free(wpipe); 732 mutex_enter(lock); 733 return (ENOMEM); /* so that caller fallback to ordinary write */ 734 } 735 736 /* Enter the loaned pages to KVA, produce new emap generation number. */ 737 uvm_emap_enter(wpipe->pipe_map.kva, pgs, npages); 738 wpipe->pipe_map.egen = uvm_emap_produce(); 739 740 /* Now we can put the pipe in direct write mode */ 741 wpipe->pipe_map.pos = bpos; 742 wpipe->pipe_map.cnt = bcnt; 743 744 /* 745 * But before we can let someone do a direct read, we 746 * have to wait until the pipe is drained. Release the 747 * pipe lock while we wait. 748 */ 749 mutex_enter(lock); 750 wpipe->pipe_state |= PIPE_DIRECTW; 751 pipeunlock(wpipe); 752 753 while (error == 0 && wpipe->pipe_buffer.cnt > 0) { 754 cv_broadcast(&wpipe->pipe_rcv); 755 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 756 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 757 error = EPIPE; 758 } 759 760 /* Pipe is drained; next read will off the direct buffer */ 761 wpipe->pipe_state |= PIPE_DIRECTR; 762 763 /* Wait until the reader is done */ 764 while (error == 0 && (wpipe->pipe_state & PIPE_DIRECTR)) { 765 cv_broadcast(&wpipe->pipe_rcv); 766 pipeselwakeup(wpipe, wpipe, POLL_IN); 767 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 768 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 769 error = EPIPE; 770 } 771 772 /* Take pipe out of direct write mode */ 773 wpipe->pipe_state &= ~(PIPE_DIRECTW | PIPE_DIRECTR); 774 775 /* Acquire the pipe lock and cleanup */ 776 (void)pipelock(wpipe, 0); 777 mutex_exit(lock); 778 779 if (pgs != NULL) { 780 /* XXX: uvm_emap_remove */ 781 uvm_unloan(pgs, npages, UVM_LOAN_TOPAGE); 782 } 783 if (error || amountpipekva > maxpipekva) 784 pipe_loan_free(wpipe); 785 786 mutex_enter(lock); 787 if (error) { 788 pipeselwakeup(wpipe, wpipe, POLL_ERR); 789 790 /* 791 * If nothing was read from what we offered, return error 792 * straight on. Otherwise update uio resid first. Caller 793 * will deal with the error condition, returning short 794 * write, error, or restarting the write(2) as appropriate. 795 */ 796 if (wpipe->pipe_map.cnt == bcnt) { 797 wpipe->pipe_map.cnt = 0; 798 cv_broadcast(&wpipe->pipe_wcv); 799 return (error); 800 } 801 802 bcnt -= wpipe->pipe_map.cnt; 803 } 804 805 uio->uio_resid -= bcnt; 806 /* uio_offset not updated, not set/used for write(2) */ 807 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + bcnt; 808 uio->uio_iov->iov_len -= bcnt; 809 if (uio->uio_iov->iov_len == 0) { 810 uio->uio_iov++; 811 uio->uio_iovcnt--; 812 } 813 814 wpipe->pipe_map.cnt = 0; 815 return (error); 816 } 817 #endif /* !PIPE_NODIRECT */ 818 819 static int 820 pipe_write(file_t *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 821 int flags) 822 { 823 struct pipe *wpipe, *rpipe; 824 struct pipebuf *bp; 825 kmutex_t *lock; 826 int error; 827 unsigned int wakeup_state = 0; 828 829 /* We want to write to our peer */ 830 rpipe = (struct pipe *) fp->f_data; 831 lock = rpipe->pipe_lock; 832 error = 0; 833 834 mutex_enter(lock); 835 wpipe = rpipe->pipe_peer; 836 837 /* 838 * Detect loss of pipe read side, issue SIGPIPE if lost. 839 */ 840 if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF) != 0) { 841 mutex_exit(lock); 842 return EPIPE; 843 } 844 ++wpipe->pipe_busy; 845 846 /* Aquire the long-term pipe lock */ 847 if ((error = pipelock(wpipe, 1)) != 0) { 848 --wpipe->pipe_busy; 849 if (wpipe->pipe_busy == 0) { 850 wpipe->pipe_state &= ~PIPE_RESTART; 851 cv_broadcast(&wpipe->pipe_draincv); 852 } 853 mutex_exit(lock); 854 return (error); 855 } 856 857 bp = &wpipe->pipe_buffer; 858 859 /* 860 * If it is advantageous to resize the pipe buffer, do so. 861 */ 862 if ((uio->uio_resid > PIPE_SIZE) && 863 (nbigpipe < maxbigpipes) && 864 #ifndef PIPE_NODIRECT 865 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 866 #endif 867 (bp->size <= PIPE_SIZE) && (bp->cnt == 0)) { 868 869 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) 870 atomic_inc_uint(&nbigpipe); 871 } 872 873 while (uio->uio_resid) { 874 size_t space; 875 876 #ifndef PIPE_NODIRECT 877 /* 878 * Pipe buffered writes cannot be coincidental with 879 * direct writes. Also, only one direct write can be 880 * in progress at any one time. We wait until the currently 881 * executing direct write is completed before continuing. 882 * 883 * We break out if a signal occurs or the reader goes away. 884 */ 885 while (error == 0 && wpipe->pipe_state & PIPE_DIRECTW) { 886 cv_broadcast(&wpipe->pipe_rcv); 887 pipeunlock(wpipe); 888 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 889 (void)pipelock(wpipe, 0); 890 if (wpipe->pipe_state & PIPE_EOF) 891 error = EPIPE; 892 } 893 if (error) 894 break; 895 896 /* 897 * If the transfer is large, we can gain performance if 898 * we do process-to-process copies directly. 899 * If the write is non-blocking, we don't use the 900 * direct write mechanism. 901 * 902 * The direct write mechanism will detect the reader going 903 * away on us. 904 */ 905 if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) && 906 (fp->f_flag & FNONBLOCK) == 0 && 907 (wpipe->pipe_map.kva || (amountpipekva < limitpipekva))) { 908 error = pipe_direct_write(fp, wpipe, uio); 909 910 /* 911 * Break out if error occurred, unless it's ENOMEM. 912 * ENOMEM means we failed to allocate some resources 913 * for direct write, so we just fallback to ordinary 914 * write. If the direct write was successful, 915 * process rest of data via ordinary write. 916 */ 917 if (error == 0) 918 continue; 919 920 if (error != ENOMEM) 921 break; 922 } 923 #endif /* PIPE_NODIRECT */ 924 925 space = bp->size - bp->cnt; 926 927 /* Writes of size <= PIPE_BUF must be atomic. */ 928 if ((space < uio->uio_resid) && (uio->uio_resid <= PIPE_BUF)) 929 space = 0; 930 931 if (space > 0) { 932 int size; /* Transfer size */ 933 int segsize; /* first segment to transfer */ 934 935 /* 936 * Transfer size is minimum of uio transfer 937 * and free space in pipe buffer. 938 */ 939 if (space > uio->uio_resid) 940 size = uio->uio_resid; 941 else 942 size = space; 943 /* 944 * First segment to transfer is minimum of 945 * transfer size and contiguous space in 946 * pipe buffer. If first segment to transfer 947 * is less than the transfer size, we've got 948 * a wraparound in the buffer. 949 */ 950 segsize = bp->size - bp->in; 951 if (segsize > size) 952 segsize = size; 953 954 /* Transfer first segment */ 955 mutex_exit(lock); 956 error = uiomove((char *)bp->buffer + bp->in, segsize, 957 uio); 958 959 if (error == 0 && segsize < size) { 960 /* 961 * Transfer remaining part now, to 962 * support atomic writes. Wraparound 963 * happened. 964 */ 965 KASSERT(bp->in + segsize == bp->size); 966 error = uiomove(bp->buffer, 967 size - segsize, uio); 968 } 969 mutex_enter(lock); 970 if (error) 971 break; 972 973 bp->in += size; 974 if (bp->in >= bp->size) { 975 KASSERT(bp->in == size - segsize + bp->size); 976 bp->in = size - segsize; 977 } 978 979 bp->cnt += size; 980 KASSERT(bp->cnt <= bp->size); 981 wakeup_state = 0; 982 } else { 983 /* 984 * If the "read-side" has been blocked, wake it up now. 985 */ 986 cv_broadcast(&wpipe->pipe_rcv); 987 988 /* 989 * Don't block on non-blocking I/O. 990 */ 991 if (fp->f_flag & FNONBLOCK) { 992 error = EAGAIN; 993 break; 994 } 995 996 /* 997 * We have no more space and have something to offer, 998 * wake up select/poll. 999 */ 1000 if (bp->cnt) 1001 pipeselwakeup(wpipe, wpipe, POLL_IN); 1002 1003 if (wakeup_state & PIPE_RESTART) { 1004 error = ERESTART; 1005 break; 1006 } 1007 1008 pipeunlock(wpipe); 1009 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 1010 (void)pipelock(wpipe, 0); 1011 if (error != 0) 1012 break; 1013 /* 1014 * If read side wants to go away, we just issue a signal 1015 * to ourselves. 1016 */ 1017 if (wpipe->pipe_state & PIPE_EOF) { 1018 error = EPIPE; 1019 break; 1020 } 1021 wakeup_state = wpipe->pipe_state; 1022 } 1023 } 1024 1025 --wpipe->pipe_busy; 1026 if (wpipe->pipe_busy == 0) { 1027 wpipe->pipe_state &= ~PIPE_RESTART; 1028 cv_broadcast(&wpipe->pipe_draincv); 1029 } 1030 if (bp->cnt > 0) { 1031 cv_broadcast(&wpipe->pipe_rcv); 1032 } 1033 1034 /* 1035 * Don't return EPIPE if I/O was successful 1036 */ 1037 if (error == EPIPE && bp->cnt == 0 && uio->uio_resid == 0) 1038 error = 0; 1039 1040 if (error == 0) 1041 getnanotime(&wpipe->pipe_mtime); 1042 1043 /* 1044 * We have something to offer, wake up select/poll. 1045 * wpipe->pipe_map.cnt is always 0 in this point (direct write 1046 * is only done synchronously), so check only wpipe->pipe_buffer.cnt 1047 */ 1048 if (bp->cnt) 1049 pipeselwakeup(wpipe, wpipe, POLL_IN); 1050 1051 /* 1052 * Arrange for next read(2) to do a signal. 1053 */ 1054 wpipe->pipe_state |= PIPE_SIGNALR; 1055 1056 pipeunlock(wpipe); 1057 mutex_exit(lock); 1058 return (error); 1059 } 1060 1061 /* 1062 * We implement a very minimal set of ioctls for compatibility with sockets. 1063 */ 1064 int 1065 pipe_ioctl(file_t *fp, u_long cmd, void *data) 1066 { 1067 struct pipe *pipe = fp->f_data; 1068 kmutex_t *lock = pipe->pipe_lock; 1069 1070 switch (cmd) { 1071 1072 case FIONBIO: 1073 return (0); 1074 1075 case FIOASYNC: 1076 mutex_enter(lock); 1077 if (*(int *)data) { 1078 pipe->pipe_state |= PIPE_ASYNC; 1079 } else { 1080 pipe->pipe_state &= ~PIPE_ASYNC; 1081 } 1082 mutex_exit(lock); 1083 return (0); 1084 1085 case FIONREAD: 1086 mutex_enter(lock); 1087 #ifndef PIPE_NODIRECT 1088 if (pipe->pipe_state & PIPE_DIRECTW) 1089 *(int *)data = pipe->pipe_map.cnt; 1090 else 1091 #endif 1092 *(int *)data = pipe->pipe_buffer.cnt; 1093 mutex_exit(lock); 1094 return (0); 1095 1096 case FIONWRITE: 1097 /* Look at other side */ 1098 pipe = pipe->pipe_peer; 1099 mutex_enter(lock); 1100 #ifndef PIPE_NODIRECT 1101 if (pipe->pipe_state & PIPE_DIRECTW) 1102 *(int *)data = pipe->pipe_map.cnt; 1103 else 1104 #endif 1105 *(int *)data = pipe->pipe_buffer.cnt; 1106 mutex_exit(lock); 1107 return (0); 1108 1109 case FIONSPACE: 1110 /* Look at other side */ 1111 pipe = pipe->pipe_peer; 1112 mutex_enter(lock); 1113 #ifndef PIPE_NODIRECT 1114 /* 1115 * If we're in direct-mode, we don't really have a 1116 * send queue, and any other write will block. Thus 1117 * zero seems like the best answer. 1118 */ 1119 if (pipe->pipe_state & PIPE_DIRECTW) 1120 *(int *)data = 0; 1121 else 1122 #endif 1123 *(int *)data = pipe->pipe_buffer.size - 1124 pipe->pipe_buffer.cnt; 1125 mutex_exit(lock); 1126 return (0); 1127 1128 case TIOCSPGRP: 1129 case FIOSETOWN: 1130 return fsetown(&pipe->pipe_pgid, cmd, data); 1131 1132 case TIOCGPGRP: 1133 case FIOGETOWN: 1134 return fgetown(pipe->pipe_pgid, cmd, data); 1135 1136 } 1137 return (EPASSTHROUGH); 1138 } 1139 1140 int 1141 pipe_poll(file_t *fp, int events) 1142 { 1143 struct pipe *rpipe = fp->f_data; 1144 struct pipe *wpipe; 1145 int eof = 0; 1146 int revents = 0; 1147 1148 mutex_enter(rpipe->pipe_lock); 1149 wpipe = rpipe->pipe_peer; 1150 1151 if (events & (POLLIN | POLLRDNORM)) 1152 if ((rpipe->pipe_buffer.cnt > 0) || 1153 #ifndef PIPE_NODIRECT 1154 (rpipe->pipe_state & PIPE_DIRECTR) || 1155 #endif 1156 (rpipe->pipe_state & PIPE_EOF)) 1157 revents |= events & (POLLIN | POLLRDNORM); 1158 1159 eof |= (rpipe->pipe_state & PIPE_EOF); 1160 1161 if (wpipe == NULL) 1162 revents |= events & (POLLOUT | POLLWRNORM); 1163 else { 1164 if (events & (POLLOUT | POLLWRNORM)) 1165 if ((wpipe->pipe_state & PIPE_EOF) || ( 1166 #ifndef PIPE_NODIRECT 1167 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 1168 #endif 1169 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 1170 revents |= events & (POLLOUT | POLLWRNORM); 1171 1172 eof |= (wpipe->pipe_state & PIPE_EOF); 1173 } 1174 1175 if (wpipe == NULL || eof) 1176 revents |= POLLHUP; 1177 1178 if (revents == 0) { 1179 if (events & (POLLIN | POLLRDNORM)) 1180 selrecord(curlwp, &rpipe->pipe_sel); 1181 1182 if (events & (POLLOUT | POLLWRNORM)) 1183 selrecord(curlwp, &wpipe->pipe_sel); 1184 } 1185 mutex_exit(rpipe->pipe_lock); 1186 1187 return (revents); 1188 } 1189 1190 static int 1191 pipe_stat(file_t *fp, struct stat *ub) 1192 { 1193 struct pipe *pipe = fp->f_data; 1194 1195 mutex_enter(pipe->pipe_lock); 1196 memset(ub, 0, sizeof(*ub)); 1197 ub->st_mode = S_IFIFO | S_IRUSR | S_IWUSR; 1198 ub->st_blksize = pipe->pipe_buffer.size; 1199 if (ub->st_blksize == 0 && pipe->pipe_peer) 1200 ub->st_blksize = pipe->pipe_peer->pipe_buffer.size; 1201 ub->st_size = pipe->pipe_buffer.cnt; 1202 ub->st_blocks = (ub->st_size) ? 1 : 0; 1203 ub->st_atimespec = pipe->pipe_atime; 1204 ub->st_mtimespec = pipe->pipe_mtime; 1205 ub->st_ctimespec = ub->st_birthtimespec = pipe->pipe_btime; 1206 ub->st_uid = kauth_cred_geteuid(fp->f_cred); 1207 ub->st_gid = kauth_cred_getegid(fp->f_cred); 1208 1209 /* 1210 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 1211 * XXX (st_dev, st_ino) should be unique. 1212 */ 1213 mutex_exit(pipe->pipe_lock); 1214 return 0; 1215 } 1216 1217 static int 1218 pipe_close(file_t *fp) 1219 { 1220 struct pipe *pipe = fp->f_data; 1221 1222 fp->f_data = NULL; 1223 pipeclose(pipe); 1224 return (0); 1225 } 1226 1227 static void 1228 pipe_restart(file_t *fp) 1229 { 1230 struct pipe *pipe = fp->f_data; 1231 1232 /* 1233 * Unblock blocked reads/writes in order to allow close() to complete. 1234 * System calls return ERESTART so that the fd is revalidated. 1235 * (Partial writes return the transfer length.) 1236 */ 1237 mutex_enter(pipe->pipe_lock); 1238 pipe->pipe_state |= PIPE_RESTART; 1239 /* Wakeup both cvs, maybe we only need one, but maybe there are some 1240 * other paths where wakeup is needed, and it saves deciding which! */ 1241 cv_broadcast(&pipe->pipe_rcv); 1242 cv_broadcast(&pipe->pipe_wcv); 1243 mutex_exit(pipe->pipe_lock); 1244 } 1245 1246 static void 1247 pipe_free_kmem(struct pipe *pipe) 1248 { 1249 1250 if (pipe->pipe_buffer.buffer != NULL) { 1251 if (pipe->pipe_buffer.size > PIPE_SIZE) { 1252 atomic_dec_uint(&nbigpipe); 1253 } 1254 if (pipe->pipe_buffer.buffer != (void *)pipe->pipe_kmem) { 1255 uvm_km_free(kernel_map, 1256 (vaddr_t)pipe->pipe_buffer.buffer, 1257 pipe->pipe_buffer.size, UVM_KMF_PAGEABLE); 1258 atomic_add_int(&amountpipekva, 1259 -pipe->pipe_buffer.size); 1260 } 1261 pipe->pipe_buffer.buffer = NULL; 1262 } 1263 #ifndef PIPE_NODIRECT 1264 if (pipe->pipe_map.kva != 0) { 1265 pipe_loan_free(pipe); 1266 pipe->pipe_map.cnt = 0; 1267 pipe->pipe_map.kva = 0; 1268 pipe->pipe_map.pos = 0; 1269 pipe->pipe_map.npages = 0; 1270 } 1271 #endif /* !PIPE_NODIRECT */ 1272 } 1273 1274 /* 1275 * Shutdown the pipe. 1276 */ 1277 static void 1278 pipeclose(struct pipe *pipe) 1279 { 1280 kmutex_t *lock; 1281 struct pipe *ppipe; 1282 1283 if (pipe == NULL) 1284 return; 1285 1286 KASSERT(cv_is_valid(&pipe->pipe_rcv)); 1287 KASSERT(cv_is_valid(&pipe->pipe_wcv)); 1288 KASSERT(cv_is_valid(&pipe->pipe_draincv)); 1289 KASSERT(cv_is_valid(&pipe->pipe_lkcv)); 1290 1291 lock = pipe->pipe_lock; 1292 if (lock == NULL) 1293 /* Must have failed during create */ 1294 goto free_resources; 1295 1296 mutex_enter(lock); 1297 pipeselwakeup(pipe, pipe, POLL_HUP); 1298 1299 /* 1300 * If the other side is blocked, wake it up saying that 1301 * we want to close it down. 1302 */ 1303 pipe->pipe_state |= PIPE_EOF; 1304 if (pipe->pipe_busy) { 1305 while (pipe->pipe_busy) { 1306 cv_broadcast(&pipe->pipe_wcv); 1307 cv_wait_sig(&pipe->pipe_draincv, lock); 1308 } 1309 } 1310 1311 /* 1312 * Disconnect from peer. 1313 */ 1314 if ((ppipe = pipe->pipe_peer) != NULL) { 1315 pipeselwakeup(ppipe, ppipe, POLL_HUP); 1316 ppipe->pipe_state |= PIPE_EOF; 1317 cv_broadcast(&ppipe->pipe_rcv); 1318 ppipe->pipe_peer = NULL; 1319 } 1320 1321 /* 1322 * Any knote objects still left in the list are 1323 * the one attached by peer. Since no one will 1324 * traverse this list, we just clear it. 1325 */ 1326 SLIST_INIT(&pipe->pipe_sel.sel_klist); 1327 1328 KASSERT((pipe->pipe_state & PIPE_LOCKFL) == 0); 1329 mutex_exit(lock); 1330 mutex_obj_free(lock); 1331 1332 /* 1333 * Free resources. 1334 */ 1335 free_resources: 1336 pipe->pipe_pgid = 0; 1337 pipe->pipe_state = PIPE_SIGNALR; 1338 pipe_free_kmem(pipe); 1339 if (pipe->pipe_kmem != 0) { 1340 pool_cache_put(pipe_rd_cache, pipe); 1341 } else { 1342 pool_cache_put(pipe_wr_cache, pipe); 1343 } 1344 } 1345 1346 static void 1347 filt_pipedetach(struct knote *kn) 1348 { 1349 struct pipe *pipe; 1350 kmutex_t *lock; 1351 1352 pipe = ((file_t *)kn->kn_obj)->f_data; 1353 lock = pipe->pipe_lock; 1354 1355 mutex_enter(lock); 1356 1357 switch(kn->kn_filter) { 1358 case EVFILT_WRITE: 1359 /* Need the peer structure, not our own. */ 1360 pipe = pipe->pipe_peer; 1361 1362 /* If reader end already closed, just return. */ 1363 if (pipe == NULL) { 1364 mutex_exit(lock); 1365 return; 1366 } 1367 1368 break; 1369 default: 1370 /* Nothing to do. */ 1371 break; 1372 } 1373 1374 KASSERT(kn->kn_hook == pipe); 1375 SLIST_REMOVE(&pipe->pipe_sel.sel_klist, kn, knote, kn_selnext); 1376 mutex_exit(lock); 1377 } 1378 1379 static int 1380 filt_piperead(struct knote *kn, long hint) 1381 { 1382 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_data; 1383 struct pipe *wpipe; 1384 1385 if ((hint & NOTE_SUBMIT) == 0) { 1386 mutex_enter(rpipe->pipe_lock); 1387 } 1388 wpipe = rpipe->pipe_peer; 1389 kn->kn_data = rpipe->pipe_buffer.cnt; 1390 1391 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1392 kn->kn_data = rpipe->pipe_map.cnt; 1393 1394 if ((rpipe->pipe_state & PIPE_EOF) || 1395 (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1396 kn->kn_flags |= EV_EOF; 1397 if ((hint & NOTE_SUBMIT) == 0) { 1398 mutex_exit(rpipe->pipe_lock); 1399 } 1400 return (1); 1401 } 1402 1403 if ((hint & NOTE_SUBMIT) == 0) { 1404 mutex_exit(rpipe->pipe_lock); 1405 } 1406 return (kn->kn_data > 0); 1407 } 1408 1409 static int 1410 filt_pipewrite(struct knote *kn, long hint) 1411 { 1412 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_data; 1413 struct pipe *wpipe; 1414 1415 if ((hint & NOTE_SUBMIT) == 0) { 1416 mutex_enter(rpipe->pipe_lock); 1417 } 1418 wpipe = rpipe->pipe_peer; 1419 1420 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1421 kn->kn_data = 0; 1422 kn->kn_flags |= EV_EOF; 1423 if ((hint & NOTE_SUBMIT) == 0) { 1424 mutex_exit(rpipe->pipe_lock); 1425 } 1426 return (1); 1427 } 1428 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1429 if (wpipe->pipe_state & PIPE_DIRECTW) 1430 kn->kn_data = 0; 1431 1432 if ((hint & NOTE_SUBMIT) == 0) { 1433 mutex_exit(rpipe->pipe_lock); 1434 } 1435 return (kn->kn_data >= PIPE_BUF); 1436 } 1437 1438 static const struct filterops pipe_rfiltops = 1439 { 1, NULL, filt_pipedetach, filt_piperead }; 1440 static const struct filterops pipe_wfiltops = 1441 { 1, NULL, filt_pipedetach, filt_pipewrite }; 1442 1443 static int 1444 pipe_kqfilter(file_t *fp, struct knote *kn) 1445 { 1446 struct pipe *pipe; 1447 kmutex_t *lock; 1448 1449 pipe = ((file_t *)kn->kn_obj)->f_data; 1450 lock = pipe->pipe_lock; 1451 1452 mutex_enter(lock); 1453 1454 switch (kn->kn_filter) { 1455 case EVFILT_READ: 1456 kn->kn_fop = &pipe_rfiltops; 1457 break; 1458 case EVFILT_WRITE: 1459 kn->kn_fop = &pipe_wfiltops; 1460 pipe = pipe->pipe_peer; 1461 if (pipe == NULL) { 1462 /* Other end of pipe has been closed. */ 1463 mutex_exit(lock); 1464 return (EBADF); 1465 } 1466 break; 1467 default: 1468 mutex_exit(lock); 1469 return (EINVAL); 1470 } 1471 1472 kn->kn_hook = pipe; 1473 SLIST_INSERT_HEAD(&pipe->pipe_sel.sel_klist, kn, kn_selnext); 1474 mutex_exit(lock); 1475 1476 return (0); 1477 } 1478 1479 /* 1480 * Handle pipe sysctls. 1481 */ 1482 SYSCTL_SETUP(sysctl_kern_pipe_setup, "sysctl kern.pipe subtree setup") 1483 { 1484 1485 sysctl_createv(clog, 0, NULL, NULL, 1486 CTLFLAG_PERMANENT, 1487 CTLTYPE_NODE, "kern", NULL, 1488 NULL, 0, NULL, 0, 1489 CTL_KERN, CTL_EOL); 1490 sysctl_createv(clog, 0, NULL, NULL, 1491 CTLFLAG_PERMANENT, 1492 CTLTYPE_NODE, "pipe", 1493 SYSCTL_DESCR("Pipe settings"), 1494 NULL, 0, NULL, 0, 1495 CTL_KERN, KERN_PIPE, CTL_EOL); 1496 1497 sysctl_createv(clog, 0, NULL, NULL, 1498 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1499 CTLTYPE_INT, "maxkvasz", 1500 SYSCTL_DESCR("Maximum amount of kernel memory to be " 1501 "used for pipes"), 1502 NULL, 0, &maxpipekva, 0, 1503 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXKVASZ, CTL_EOL); 1504 sysctl_createv(clog, 0, NULL, NULL, 1505 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1506 CTLTYPE_INT, "maxloankvasz", 1507 SYSCTL_DESCR("Limit for direct transfers via page loan"), 1508 NULL, 0, &limitpipekva, 0, 1509 CTL_KERN, KERN_PIPE, KERN_PIPE_LIMITKVA, CTL_EOL); 1510 sysctl_createv(clog, 0, NULL, NULL, 1511 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1512 CTLTYPE_INT, "maxbigpipes", 1513 SYSCTL_DESCR("Maximum number of \"big\" pipes"), 1514 NULL, 0, &maxbigpipes, 0, 1515 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXBIGPIPES, CTL_EOL); 1516 sysctl_createv(clog, 0, NULL, NULL, 1517 CTLFLAG_PERMANENT, 1518 CTLTYPE_INT, "nbigpipes", 1519 SYSCTL_DESCR("Number of \"big\" pipes"), 1520 NULL, 0, &nbigpipe, 0, 1521 CTL_KERN, KERN_PIPE, KERN_PIPE_NBIGPIPES, CTL_EOL); 1522 sysctl_createv(clog, 0, NULL, NULL, 1523 CTLFLAG_PERMANENT, 1524 CTLTYPE_INT, "kvasize", 1525 SYSCTL_DESCR("Amount of kernel memory consumed by pipe " 1526 "buffers"), 1527 NULL, 0, &amountpipekva, 0, 1528 CTL_KERN, KERN_PIPE, KERN_PIPE_KVASIZE, CTL_EOL); 1529 } 1530