1 /* $NetBSD: sys_pipe.c,v 1.135 2012/01/25 00:28:36 christos Exp $ */ 2 3 /*- 4 * Copyright (c) 2003, 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Paul Kranenburg, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1996 John S. Dyson 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice immediately at the beginning of the file, without modification, 41 * this list of conditions, and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. Absolutely no warranty of function or purpose is made by the author 46 * John S. Dyson. 47 * 4. Modifications may be freely made to this file if the above conditions 48 * are met. 49 */ 50 51 /* 52 * This file contains a high-performance replacement for the socket-based 53 * pipes scheme originally used. It does not support all features of 54 * sockets, but does do everything that pipes normally do. 55 * 56 * This code has two modes of operation, a small write mode and a large 57 * write mode. The small write mode acts like conventional pipes with 58 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 59 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 60 * and PIPE_SIZE in size it is mapped read-only into the kernel address space 61 * using the UVM page loan facility from where the receiving process can copy 62 * the data directly from the pages in the sending process. 63 * 64 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 65 * happen for small transfers so that the system will not spend all of 66 * its time context switching. PIPE_SIZE is constrained by the 67 * amount of kernel virtual memory. 68 */ 69 70 #include <sys/cdefs.h> 71 __KERNEL_RCSID(0, "$NetBSD: sys_pipe.c,v 1.135 2012/01/25 00:28:36 christos Exp $"); 72 73 #include <sys/param.h> 74 #include <sys/systm.h> 75 #include <sys/proc.h> 76 #include <sys/fcntl.h> 77 #include <sys/file.h> 78 #include <sys/filedesc.h> 79 #include <sys/filio.h> 80 #include <sys/kernel.h> 81 #include <sys/ttycom.h> 82 #include <sys/stat.h> 83 #include <sys/poll.h> 84 #include <sys/signalvar.h> 85 #include <sys/vnode.h> 86 #include <sys/uio.h> 87 #include <sys/select.h> 88 #include <sys/mount.h> 89 #include <sys/syscallargs.h> 90 #include <sys/sysctl.h> 91 #include <sys/kauth.h> 92 #include <sys/atomic.h> 93 #include <sys/pipe.h> 94 95 #include <uvm/uvm_extern.h> 96 97 /* 98 * Use this to disable direct I/O and decrease the code size: 99 * #define PIPE_NODIRECT 100 */ 101 102 /* XXX Disabled for now; rare hangs switching between direct/buffered */ 103 #define PIPE_NODIRECT 104 105 static int pipe_read(file_t *, off_t *, struct uio *, kauth_cred_t, int); 106 static int pipe_write(file_t *, off_t *, struct uio *, kauth_cred_t, int); 107 static int pipe_close(file_t *); 108 static int pipe_poll(file_t *, int); 109 static int pipe_kqfilter(file_t *, struct knote *); 110 static int pipe_stat(file_t *, struct stat *); 111 static int pipe_ioctl(file_t *, u_long, void *); 112 static void pipe_restart(file_t *); 113 114 static const struct fileops pipeops = { 115 .fo_read = pipe_read, 116 .fo_write = pipe_write, 117 .fo_ioctl = pipe_ioctl, 118 .fo_fcntl = fnullop_fcntl, 119 .fo_poll = pipe_poll, 120 .fo_stat = pipe_stat, 121 .fo_close = pipe_close, 122 .fo_kqfilter = pipe_kqfilter, 123 .fo_restart = pipe_restart, 124 }; 125 126 /* 127 * Default pipe buffer size(s), this can be kind-of large now because pipe 128 * space is pageable. The pipe code will try to maintain locality of 129 * reference for performance reasons, so small amounts of outstanding I/O 130 * will not wipe the cache. 131 */ 132 #define MINPIPESIZE (PIPE_SIZE / 3) 133 #define MAXPIPESIZE (2 * PIPE_SIZE / 3) 134 135 /* 136 * Maximum amount of kva for pipes -- this is kind-of a soft limit, but 137 * is there so that on large systems, we don't exhaust it. 138 */ 139 #define MAXPIPEKVA (8 * 1024 * 1024) 140 static u_int maxpipekva = MAXPIPEKVA; 141 142 /* 143 * Limit for direct transfers, we cannot, of course limit 144 * the amount of kva for pipes in general though. 145 */ 146 #define LIMITPIPEKVA (16 * 1024 * 1024) 147 static u_int limitpipekva = LIMITPIPEKVA; 148 149 /* 150 * Limit the number of "big" pipes 151 */ 152 #define LIMITBIGPIPES 32 153 static u_int maxbigpipes = LIMITBIGPIPES; 154 static u_int nbigpipe = 0; 155 156 /* 157 * Amount of KVA consumed by pipe buffers. 158 */ 159 static u_int amountpipekva = 0; 160 161 static void pipeclose(struct pipe *); 162 static void pipe_free_kmem(struct pipe *); 163 static int pipe_create(struct pipe **, pool_cache_t); 164 static int pipelock(struct pipe *, int); 165 static inline void pipeunlock(struct pipe *); 166 static void pipeselwakeup(struct pipe *, struct pipe *, int); 167 #ifndef PIPE_NODIRECT 168 static int pipe_direct_write(file_t *, struct pipe *, struct uio *); 169 #endif 170 static int pipespace(struct pipe *, int); 171 static int pipe_ctor(void *, void *, int); 172 static void pipe_dtor(void *, void *); 173 174 #ifndef PIPE_NODIRECT 175 static int pipe_loan_alloc(struct pipe *, int); 176 static void pipe_loan_free(struct pipe *); 177 #endif /* PIPE_NODIRECT */ 178 179 static pool_cache_t pipe_wr_cache; 180 static pool_cache_t pipe_rd_cache; 181 182 void 183 pipe_init(void) 184 { 185 186 /* Writer side is not automatically allocated KVA. */ 187 pipe_wr_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "pipewr", 188 NULL, IPL_NONE, pipe_ctor, pipe_dtor, NULL); 189 KASSERT(pipe_wr_cache != NULL); 190 191 /* Reader side gets preallocated KVA. */ 192 pipe_rd_cache = pool_cache_init(sizeof(struct pipe), 0, 0, 0, "piperd", 193 NULL, IPL_NONE, pipe_ctor, pipe_dtor, (void *)1); 194 KASSERT(pipe_rd_cache != NULL); 195 } 196 197 static int 198 pipe_ctor(void *arg, void *obj, int flags) 199 { 200 struct pipe *pipe; 201 vaddr_t va; 202 203 pipe = obj; 204 205 memset(pipe, 0, sizeof(struct pipe)); 206 if (arg != NULL) { 207 /* Preallocate space. */ 208 va = uvm_km_alloc(kernel_map, PIPE_SIZE, 0, 209 UVM_KMF_PAGEABLE | UVM_KMF_WAITVA); 210 KASSERT(va != 0); 211 pipe->pipe_kmem = va; 212 atomic_add_int(&amountpipekva, PIPE_SIZE); 213 } 214 cv_init(&pipe->pipe_rcv, "pipe_rd"); 215 cv_init(&pipe->pipe_wcv, "pipe_wr"); 216 cv_init(&pipe->pipe_draincv, "pipe_drn"); 217 cv_init(&pipe->pipe_lkcv, "pipe_lk"); 218 selinit(&pipe->pipe_sel); 219 pipe->pipe_state = PIPE_SIGNALR; 220 221 return 0; 222 } 223 224 static void 225 pipe_dtor(void *arg, void *obj) 226 { 227 struct pipe *pipe; 228 229 pipe = obj; 230 231 cv_destroy(&pipe->pipe_rcv); 232 cv_destroy(&pipe->pipe_wcv); 233 cv_destroy(&pipe->pipe_draincv); 234 cv_destroy(&pipe->pipe_lkcv); 235 seldestroy(&pipe->pipe_sel); 236 if (pipe->pipe_kmem != 0) { 237 uvm_km_free(kernel_map, pipe->pipe_kmem, PIPE_SIZE, 238 UVM_KMF_PAGEABLE); 239 atomic_add_int(&amountpipekva, -PIPE_SIZE); 240 } 241 } 242 243 /* 244 * The pipe system call for the DTYPE_PIPE type of pipes 245 */ 246 int 247 pipe1(struct lwp *l, register_t *retval, int flags) 248 { 249 struct pipe *rpipe, *wpipe; 250 file_t *rf, *wf; 251 int fd, error; 252 proc_t *p; 253 254 if (flags & ~(O_CLOEXEC|O_NONBLOCK|O_NOSIGPIPE)) 255 return EINVAL; 256 p = curproc; 257 rpipe = wpipe = NULL; 258 if ((error = pipe_create(&rpipe, pipe_rd_cache)) || 259 (error = pipe_create(&wpipe, pipe_wr_cache))) { 260 goto free2; 261 } 262 rpipe->pipe_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 263 wpipe->pipe_lock = rpipe->pipe_lock; 264 mutex_obj_hold(wpipe->pipe_lock); 265 266 error = fd_allocfile(&rf, &fd); 267 if (error) 268 goto free2; 269 retval[0] = fd; 270 rf->f_flag = FREAD | flags; 271 rf->f_type = DTYPE_PIPE; 272 rf->f_data = (void *)rpipe; 273 rf->f_ops = &pipeops; 274 fd_set_exclose(l, fd, (flags & O_CLOEXEC) != 0); 275 276 error = fd_allocfile(&wf, &fd); 277 if (error) 278 goto free3; 279 retval[1] = fd; 280 wf->f_flag = FWRITE | flags; 281 wf->f_type = DTYPE_PIPE; 282 wf->f_data = (void *)wpipe; 283 wf->f_ops = &pipeops; 284 fd_set_exclose(l, fd, (flags & O_CLOEXEC) != 0); 285 286 rpipe->pipe_peer = wpipe; 287 wpipe->pipe_peer = rpipe; 288 289 fd_affix(p, rf, (int)retval[0]); 290 fd_affix(p, wf, (int)retval[1]); 291 return (0); 292 free3: 293 fd_abort(p, rf, (int)retval[0]); 294 free2: 295 pipeclose(wpipe); 296 pipeclose(rpipe); 297 298 return (error); 299 } 300 301 /* 302 * Allocate kva for pipe circular buffer, the space is pageable 303 * This routine will 'realloc' the size of a pipe safely, if it fails 304 * it will retain the old buffer. 305 * If it fails it will return ENOMEM. 306 */ 307 static int 308 pipespace(struct pipe *pipe, int size) 309 { 310 void *buffer; 311 312 /* 313 * Allocate pageable virtual address space. Physical memory is 314 * allocated on demand. 315 */ 316 if (size == PIPE_SIZE && pipe->pipe_kmem != 0) { 317 buffer = (void *)pipe->pipe_kmem; 318 } else { 319 buffer = (void *)uvm_km_alloc(kernel_map, round_page(size), 320 0, UVM_KMF_PAGEABLE); 321 if (buffer == NULL) 322 return (ENOMEM); 323 atomic_add_int(&amountpipekva, size); 324 } 325 326 /* free old resources if we're resizing */ 327 pipe_free_kmem(pipe); 328 pipe->pipe_buffer.buffer = buffer; 329 pipe->pipe_buffer.size = size; 330 pipe->pipe_buffer.in = 0; 331 pipe->pipe_buffer.out = 0; 332 pipe->pipe_buffer.cnt = 0; 333 return (0); 334 } 335 336 /* 337 * Initialize and allocate VM and memory for pipe. 338 */ 339 static int 340 pipe_create(struct pipe **pipep, pool_cache_t cache) 341 { 342 struct pipe *pipe; 343 int error; 344 345 pipe = pool_cache_get(cache, PR_WAITOK); 346 KASSERT(pipe != NULL); 347 *pipep = pipe; 348 error = 0; 349 getnanotime(&pipe->pipe_btime); 350 pipe->pipe_atime = pipe->pipe_mtime = pipe->pipe_btime; 351 pipe->pipe_lock = NULL; 352 if (cache == pipe_rd_cache) { 353 error = pipespace(pipe, PIPE_SIZE); 354 } else { 355 pipe->pipe_buffer.buffer = NULL; 356 pipe->pipe_buffer.size = 0; 357 pipe->pipe_buffer.in = 0; 358 pipe->pipe_buffer.out = 0; 359 pipe->pipe_buffer.cnt = 0; 360 } 361 return error; 362 } 363 364 /* 365 * Lock a pipe for I/O, blocking other access 366 * Called with pipe spin lock held. 367 */ 368 static int 369 pipelock(struct pipe *pipe, int catch) 370 { 371 int error; 372 373 KASSERT(mutex_owned(pipe->pipe_lock)); 374 375 while (pipe->pipe_state & PIPE_LOCKFL) { 376 pipe->pipe_state |= PIPE_LWANT; 377 if (catch) { 378 error = cv_wait_sig(&pipe->pipe_lkcv, pipe->pipe_lock); 379 if (error != 0) 380 return error; 381 } else 382 cv_wait(&pipe->pipe_lkcv, pipe->pipe_lock); 383 } 384 385 pipe->pipe_state |= PIPE_LOCKFL; 386 387 return 0; 388 } 389 390 /* 391 * unlock a pipe I/O lock 392 */ 393 static inline void 394 pipeunlock(struct pipe *pipe) 395 { 396 397 KASSERT(pipe->pipe_state & PIPE_LOCKFL); 398 399 pipe->pipe_state &= ~PIPE_LOCKFL; 400 if (pipe->pipe_state & PIPE_LWANT) { 401 pipe->pipe_state &= ~PIPE_LWANT; 402 cv_broadcast(&pipe->pipe_lkcv); 403 } 404 } 405 406 /* 407 * Select/poll wakup. This also sends SIGIO to peer connected to 408 * 'sigpipe' side of pipe. 409 */ 410 static void 411 pipeselwakeup(struct pipe *selp, struct pipe *sigp, int code) 412 { 413 int band; 414 415 switch (code) { 416 case POLL_IN: 417 band = POLLIN|POLLRDNORM; 418 break; 419 case POLL_OUT: 420 band = POLLOUT|POLLWRNORM; 421 break; 422 case POLL_HUP: 423 band = POLLHUP; 424 break; 425 case POLL_ERR: 426 band = POLLERR; 427 break; 428 default: 429 band = 0; 430 #ifdef DIAGNOSTIC 431 printf("bad siginfo code %d in pipe notification.\n", code); 432 #endif 433 break; 434 } 435 436 selnotify(&selp->pipe_sel, band, NOTE_SUBMIT); 437 438 if (sigp == NULL || (sigp->pipe_state & PIPE_ASYNC) == 0) 439 return; 440 441 fownsignal(sigp->pipe_pgid, SIGIO, code, band, selp); 442 } 443 444 static int 445 pipe_read(file_t *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 446 int flags) 447 { 448 struct pipe *rpipe = (struct pipe *) fp->f_data; 449 struct pipebuf *bp = &rpipe->pipe_buffer; 450 kmutex_t *lock = rpipe->pipe_lock; 451 int error; 452 size_t nread = 0; 453 size_t size; 454 size_t ocnt; 455 unsigned int wakeup_state = 0; 456 457 mutex_enter(lock); 458 ++rpipe->pipe_busy; 459 ocnt = bp->cnt; 460 461 again: 462 error = pipelock(rpipe, 1); 463 if (error) 464 goto unlocked_error; 465 466 while (uio->uio_resid) { 467 /* 468 * Normal pipe buffer receive. 469 */ 470 if (bp->cnt > 0) { 471 size = bp->size - bp->out; 472 if (size > bp->cnt) 473 size = bp->cnt; 474 if (size > uio->uio_resid) 475 size = uio->uio_resid; 476 477 mutex_exit(lock); 478 error = uiomove((char *)bp->buffer + bp->out, size, uio); 479 mutex_enter(lock); 480 if (error) 481 break; 482 483 bp->out += size; 484 if (bp->out >= bp->size) 485 bp->out = 0; 486 487 bp->cnt -= size; 488 489 /* 490 * If there is no more to read in the pipe, reset 491 * its pointers to the beginning. This improves 492 * cache hit stats. 493 */ 494 if (bp->cnt == 0) { 495 bp->in = 0; 496 bp->out = 0; 497 } 498 nread += size; 499 continue; 500 } 501 502 #ifndef PIPE_NODIRECT 503 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) { 504 /* 505 * Direct copy, bypassing a kernel buffer. 506 */ 507 void *va; 508 u_int gen; 509 510 KASSERT(rpipe->pipe_state & PIPE_DIRECTW); 511 512 size = rpipe->pipe_map.cnt; 513 if (size > uio->uio_resid) 514 size = uio->uio_resid; 515 516 va = (char *)rpipe->pipe_map.kva + rpipe->pipe_map.pos; 517 gen = rpipe->pipe_map.egen; 518 mutex_exit(lock); 519 520 /* 521 * Consume emap and read the data from loaned pages. 522 */ 523 uvm_emap_consume(gen); 524 error = uiomove(va, size, uio); 525 526 mutex_enter(lock); 527 if (error) 528 break; 529 nread += size; 530 rpipe->pipe_map.pos += size; 531 rpipe->pipe_map.cnt -= size; 532 if (rpipe->pipe_map.cnt == 0) { 533 rpipe->pipe_state &= ~PIPE_DIRECTR; 534 cv_broadcast(&rpipe->pipe_wcv); 535 } 536 continue; 537 } 538 #endif 539 /* 540 * Break if some data was read. 541 */ 542 if (nread > 0) 543 break; 544 545 /* 546 * Detect EOF condition. 547 * Read returns 0 on EOF, no need to set error. 548 */ 549 if (rpipe->pipe_state & PIPE_EOF) 550 break; 551 552 /* 553 * Don't block on non-blocking I/O. 554 */ 555 if (fp->f_flag & FNONBLOCK) { 556 error = EAGAIN; 557 break; 558 } 559 560 /* 561 * Unlock the pipe buffer for our remaining processing. 562 * We will either break out with an error or we will 563 * sleep and relock to loop. 564 */ 565 pipeunlock(rpipe); 566 567 /* 568 * Re-check to see if more direct writes are pending. 569 */ 570 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) 571 goto again; 572 573 #if 1 /* XXX (dsl) I'm sure these aren't needed here ... */ 574 /* 575 * We want to read more, wake up select/poll. 576 */ 577 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 578 579 /* 580 * If the "write-side" is blocked, wake it up now. 581 */ 582 cv_broadcast(&rpipe->pipe_wcv); 583 #endif 584 585 if (wakeup_state & PIPE_RESTART) { 586 error = ERESTART; 587 goto unlocked_error; 588 } 589 590 /* Now wait until the pipe is filled */ 591 error = cv_wait_sig(&rpipe->pipe_rcv, lock); 592 if (error != 0) 593 goto unlocked_error; 594 wakeup_state = rpipe->pipe_state; 595 goto again; 596 } 597 598 if (error == 0) 599 getnanotime(&rpipe->pipe_atime); 600 pipeunlock(rpipe); 601 602 unlocked_error: 603 --rpipe->pipe_busy; 604 if (rpipe->pipe_busy == 0) { 605 rpipe->pipe_state &= ~PIPE_RESTART; 606 cv_broadcast(&rpipe->pipe_draincv); 607 } 608 if (bp->cnt < MINPIPESIZE) { 609 cv_broadcast(&rpipe->pipe_wcv); 610 } 611 612 /* 613 * If anything was read off the buffer, signal to the writer it's 614 * possible to write more data. Also send signal if we are here for the 615 * first time after last write. 616 */ 617 if ((bp->size - bp->cnt) >= PIPE_BUF 618 && (ocnt != bp->cnt || (rpipe->pipe_state & PIPE_SIGNALR))) { 619 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 620 rpipe->pipe_state &= ~PIPE_SIGNALR; 621 } 622 623 mutex_exit(lock); 624 return (error); 625 } 626 627 #ifndef PIPE_NODIRECT 628 /* 629 * Allocate structure for loan transfer. 630 */ 631 static int 632 pipe_loan_alloc(struct pipe *wpipe, int npages) 633 { 634 vsize_t len; 635 636 len = (vsize_t)npages << PAGE_SHIFT; 637 atomic_add_int(&amountpipekva, len); 638 wpipe->pipe_map.kva = uvm_km_alloc(kernel_map, len, 0, 639 UVM_KMF_VAONLY | UVM_KMF_WAITVA); 640 if (wpipe->pipe_map.kva == 0) { 641 atomic_add_int(&amountpipekva, -len); 642 return (ENOMEM); 643 } 644 645 wpipe->pipe_map.npages = npages; 646 wpipe->pipe_map.pgs = kmem_alloc(npages * sizeof(struct vm_page *), 647 KM_SLEEP); 648 return (0); 649 } 650 651 /* 652 * Free resources allocated for loan transfer. 653 */ 654 static void 655 pipe_loan_free(struct pipe *wpipe) 656 { 657 vsize_t len; 658 659 len = (vsize_t)wpipe->pipe_map.npages << PAGE_SHIFT; 660 uvm_emap_remove(wpipe->pipe_map.kva, len); /* XXX */ 661 uvm_km_free(kernel_map, wpipe->pipe_map.kva, len, UVM_KMF_VAONLY); 662 wpipe->pipe_map.kva = 0; 663 atomic_add_int(&amountpipekva, -len); 664 kmem_free(wpipe->pipe_map.pgs, 665 wpipe->pipe_map.npages * sizeof(struct vm_page *)); 666 wpipe->pipe_map.pgs = NULL; 667 } 668 669 /* 670 * NetBSD direct write, using uvm_loan() mechanism. 671 * This implements the pipe buffer write mechanism. Note that only 672 * a direct write OR a normal pipe write can be pending at any given time. 673 * If there are any characters in the pipe buffer, the direct write will 674 * be deferred until the receiving process grabs all of the bytes from 675 * the pipe buffer. Then the direct mapping write is set-up. 676 * 677 * Called with the long-term pipe lock held. 678 */ 679 static int 680 pipe_direct_write(file_t *fp, struct pipe *wpipe, struct uio *uio) 681 { 682 struct vm_page **pgs; 683 vaddr_t bbase, base, bend; 684 vsize_t blen, bcnt; 685 int error, npages; 686 voff_t bpos; 687 kmutex_t *lock = wpipe->pipe_lock; 688 689 KASSERT(mutex_owned(wpipe->pipe_lock)); 690 KASSERT(wpipe->pipe_map.cnt == 0); 691 692 mutex_exit(lock); 693 694 /* 695 * Handle first PIPE_CHUNK_SIZE bytes of buffer. Deal with buffers 696 * not aligned to PAGE_SIZE. 697 */ 698 bbase = (vaddr_t)uio->uio_iov->iov_base; 699 base = trunc_page(bbase); 700 bend = round_page(bbase + uio->uio_iov->iov_len); 701 blen = bend - base; 702 bpos = bbase - base; 703 704 if (blen > PIPE_DIRECT_CHUNK) { 705 blen = PIPE_DIRECT_CHUNK; 706 bend = base + blen; 707 bcnt = PIPE_DIRECT_CHUNK - bpos; 708 } else { 709 bcnt = uio->uio_iov->iov_len; 710 } 711 npages = blen >> PAGE_SHIFT; 712 713 /* 714 * Free the old kva if we need more pages than we have 715 * allocated. 716 */ 717 if (wpipe->pipe_map.kva != 0 && npages > wpipe->pipe_map.npages) 718 pipe_loan_free(wpipe); 719 720 /* Allocate new kva. */ 721 if (wpipe->pipe_map.kva == 0) { 722 error = pipe_loan_alloc(wpipe, npages); 723 if (error) { 724 mutex_enter(lock); 725 return (error); 726 } 727 } 728 729 /* Loan the write buffer memory from writer process */ 730 pgs = wpipe->pipe_map.pgs; 731 error = uvm_loan(&uio->uio_vmspace->vm_map, base, blen, 732 pgs, UVM_LOAN_TOPAGE); 733 if (error) { 734 pipe_loan_free(wpipe); 735 mutex_enter(lock); 736 return (ENOMEM); /* so that caller fallback to ordinary write */ 737 } 738 739 /* Enter the loaned pages to KVA, produce new emap generation number. */ 740 uvm_emap_enter(wpipe->pipe_map.kva, pgs, npages); 741 wpipe->pipe_map.egen = uvm_emap_produce(); 742 743 /* Now we can put the pipe in direct write mode */ 744 wpipe->pipe_map.pos = bpos; 745 wpipe->pipe_map.cnt = bcnt; 746 747 /* 748 * But before we can let someone do a direct read, we 749 * have to wait until the pipe is drained. Release the 750 * pipe lock while we wait. 751 */ 752 mutex_enter(lock); 753 wpipe->pipe_state |= PIPE_DIRECTW; 754 pipeunlock(wpipe); 755 756 while (error == 0 && wpipe->pipe_buffer.cnt > 0) { 757 cv_broadcast(&wpipe->pipe_rcv); 758 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 759 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 760 error = EPIPE; 761 } 762 763 /* Pipe is drained; next read will off the direct buffer */ 764 wpipe->pipe_state |= PIPE_DIRECTR; 765 766 /* Wait until the reader is done */ 767 while (error == 0 && (wpipe->pipe_state & PIPE_DIRECTR)) { 768 cv_broadcast(&wpipe->pipe_rcv); 769 pipeselwakeup(wpipe, wpipe, POLL_IN); 770 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 771 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 772 error = EPIPE; 773 } 774 775 /* Take pipe out of direct write mode */ 776 wpipe->pipe_state &= ~(PIPE_DIRECTW | PIPE_DIRECTR); 777 778 /* Acquire the pipe lock and cleanup */ 779 (void)pipelock(wpipe, 0); 780 mutex_exit(lock); 781 782 if (pgs != NULL) { 783 /* XXX: uvm_emap_remove */ 784 uvm_unloan(pgs, npages, UVM_LOAN_TOPAGE); 785 } 786 if (error || amountpipekva > maxpipekva) 787 pipe_loan_free(wpipe); 788 789 mutex_enter(lock); 790 if (error) { 791 pipeselwakeup(wpipe, wpipe, POLL_ERR); 792 793 /* 794 * If nothing was read from what we offered, return error 795 * straight on. Otherwise update uio resid first. Caller 796 * will deal with the error condition, returning short 797 * write, error, or restarting the write(2) as appropriate. 798 */ 799 if (wpipe->pipe_map.cnt == bcnt) { 800 wpipe->pipe_map.cnt = 0; 801 cv_broadcast(&wpipe->pipe_wcv); 802 return (error); 803 } 804 805 bcnt -= wpipe->pipe_map.cnt; 806 } 807 808 uio->uio_resid -= bcnt; 809 /* uio_offset not updated, not set/used for write(2) */ 810 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + bcnt; 811 uio->uio_iov->iov_len -= bcnt; 812 if (uio->uio_iov->iov_len == 0) { 813 uio->uio_iov++; 814 uio->uio_iovcnt--; 815 } 816 817 wpipe->pipe_map.cnt = 0; 818 return (error); 819 } 820 #endif /* !PIPE_NODIRECT */ 821 822 static int 823 pipe_write(file_t *fp, off_t *offset, struct uio *uio, kauth_cred_t cred, 824 int flags) 825 { 826 struct pipe *wpipe, *rpipe; 827 struct pipebuf *bp; 828 kmutex_t *lock; 829 int error; 830 unsigned int wakeup_state = 0; 831 832 /* We want to write to our peer */ 833 rpipe = (struct pipe *) fp->f_data; 834 lock = rpipe->pipe_lock; 835 error = 0; 836 837 mutex_enter(lock); 838 wpipe = rpipe->pipe_peer; 839 840 /* 841 * Detect loss of pipe read side, issue SIGPIPE if lost. 842 */ 843 if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF) != 0) { 844 mutex_exit(lock); 845 return EPIPE; 846 } 847 ++wpipe->pipe_busy; 848 849 /* Aquire the long-term pipe lock */ 850 if ((error = pipelock(wpipe, 1)) != 0) { 851 --wpipe->pipe_busy; 852 if (wpipe->pipe_busy == 0) { 853 wpipe->pipe_state &= ~PIPE_RESTART; 854 cv_broadcast(&wpipe->pipe_draincv); 855 } 856 mutex_exit(lock); 857 return (error); 858 } 859 860 bp = &wpipe->pipe_buffer; 861 862 /* 863 * If it is advantageous to resize the pipe buffer, do so. 864 */ 865 if ((uio->uio_resid > PIPE_SIZE) && 866 (nbigpipe < maxbigpipes) && 867 #ifndef PIPE_NODIRECT 868 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 869 #endif 870 (bp->size <= PIPE_SIZE) && (bp->cnt == 0)) { 871 872 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) 873 atomic_inc_uint(&nbigpipe); 874 } 875 876 while (uio->uio_resid) { 877 size_t space; 878 879 #ifndef PIPE_NODIRECT 880 /* 881 * Pipe buffered writes cannot be coincidental with 882 * direct writes. Also, only one direct write can be 883 * in progress at any one time. We wait until the currently 884 * executing direct write is completed before continuing. 885 * 886 * We break out if a signal occurs or the reader goes away. 887 */ 888 while (error == 0 && wpipe->pipe_state & PIPE_DIRECTW) { 889 cv_broadcast(&wpipe->pipe_rcv); 890 pipeunlock(wpipe); 891 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 892 (void)pipelock(wpipe, 0); 893 if (wpipe->pipe_state & PIPE_EOF) 894 error = EPIPE; 895 } 896 if (error) 897 break; 898 899 /* 900 * If the transfer is large, we can gain performance if 901 * we do process-to-process copies directly. 902 * If the write is non-blocking, we don't use the 903 * direct write mechanism. 904 * 905 * The direct write mechanism will detect the reader going 906 * away on us. 907 */ 908 if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) && 909 (fp->f_flag & FNONBLOCK) == 0 && 910 (wpipe->pipe_map.kva || (amountpipekva < limitpipekva))) { 911 error = pipe_direct_write(fp, wpipe, uio); 912 913 /* 914 * Break out if error occurred, unless it's ENOMEM. 915 * ENOMEM means we failed to allocate some resources 916 * for direct write, so we just fallback to ordinary 917 * write. If the direct write was successful, 918 * process rest of data via ordinary write. 919 */ 920 if (error == 0) 921 continue; 922 923 if (error != ENOMEM) 924 break; 925 } 926 #endif /* PIPE_NODIRECT */ 927 928 space = bp->size - bp->cnt; 929 930 /* Writes of size <= PIPE_BUF must be atomic. */ 931 if ((space < uio->uio_resid) && (uio->uio_resid <= PIPE_BUF)) 932 space = 0; 933 934 if (space > 0) { 935 int size; /* Transfer size */ 936 int segsize; /* first segment to transfer */ 937 938 /* 939 * Transfer size is minimum of uio transfer 940 * and free space in pipe buffer. 941 */ 942 if (space > uio->uio_resid) 943 size = uio->uio_resid; 944 else 945 size = space; 946 /* 947 * First segment to transfer is minimum of 948 * transfer size and contiguous space in 949 * pipe buffer. If first segment to transfer 950 * is less than the transfer size, we've got 951 * a wraparound in the buffer. 952 */ 953 segsize = bp->size - bp->in; 954 if (segsize > size) 955 segsize = size; 956 957 /* Transfer first segment */ 958 mutex_exit(lock); 959 error = uiomove((char *)bp->buffer + bp->in, segsize, 960 uio); 961 962 if (error == 0 && segsize < size) { 963 /* 964 * Transfer remaining part now, to 965 * support atomic writes. Wraparound 966 * happened. 967 */ 968 KASSERT(bp->in + segsize == bp->size); 969 error = uiomove(bp->buffer, 970 size - segsize, uio); 971 } 972 mutex_enter(lock); 973 if (error) 974 break; 975 976 bp->in += size; 977 if (bp->in >= bp->size) { 978 KASSERT(bp->in == size - segsize + bp->size); 979 bp->in = size - segsize; 980 } 981 982 bp->cnt += size; 983 KASSERT(bp->cnt <= bp->size); 984 wakeup_state = 0; 985 } else { 986 /* 987 * If the "read-side" has been blocked, wake it up now. 988 */ 989 cv_broadcast(&wpipe->pipe_rcv); 990 991 /* 992 * Don't block on non-blocking I/O. 993 */ 994 if (fp->f_flag & FNONBLOCK) { 995 error = EAGAIN; 996 break; 997 } 998 999 /* 1000 * We have no more space and have something to offer, 1001 * wake up select/poll. 1002 */ 1003 if (bp->cnt) 1004 pipeselwakeup(wpipe, wpipe, POLL_IN); 1005 1006 if (wakeup_state & PIPE_RESTART) { 1007 error = ERESTART; 1008 break; 1009 } 1010 1011 pipeunlock(wpipe); 1012 error = cv_wait_sig(&wpipe->pipe_wcv, lock); 1013 (void)pipelock(wpipe, 0); 1014 if (error != 0) 1015 break; 1016 /* 1017 * If read side wants to go away, we just issue a signal 1018 * to ourselves. 1019 */ 1020 if (wpipe->pipe_state & PIPE_EOF) { 1021 error = EPIPE; 1022 break; 1023 } 1024 wakeup_state = wpipe->pipe_state; 1025 } 1026 } 1027 1028 --wpipe->pipe_busy; 1029 if (wpipe->pipe_busy == 0) { 1030 wpipe->pipe_state &= ~PIPE_RESTART; 1031 cv_broadcast(&wpipe->pipe_draincv); 1032 } 1033 if (bp->cnt > 0) { 1034 cv_broadcast(&wpipe->pipe_rcv); 1035 } 1036 1037 /* 1038 * Don't return EPIPE if I/O was successful 1039 */ 1040 if (error == EPIPE && bp->cnt == 0 && uio->uio_resid == 0) 1041 error = 0; 1042 1043 if (error == 0) 1044 getnanotime(&wpipe->pipe_mtime); 1045 1046 /* 1047 * We have something to offer, wake up select/poll. 1048 * wpipe->pipe_map.cnt is always 0 in this point (direct write 1049 * is only done synchronously), so check only wpipe->pipe_buffer.cnt 1050 */ 1051 if (bp->cnt) 1052 pipeselwakeup(wpipe, wpipe, POLL_IN); 1053 1054 /* 1055 * Arrange for next read(2) to do a signal. 1056 */ 1057 wpipe->pipe_state |= PIPE_SIGNALR; 1058 1059 pipeunlock(wpipe); 1060 mutex_exit(lock); 1061 return (error); 1062 } 1063 1064 /* 1065 * We implement a very minimal set of ioctls for compatibility with sockets. 1066 */ 1067 int 1068 pipe_ioctl(file_t *fp, u_long cmd, void *data) 1069 { 1070 struct pipe *pipe = fp->f_data; 1071 kmutex_t *lock = pipe->pipe_lock; 1072 1073 switch (cmd) { 1074 1075 case FIONBIO: 1076 return (0); 1077 1078 case FIOASYNC: 1079 mutex_enter(lock); 1080 if (*(int *)data) { 1081 pipe->pipe_state |= PIPE_ASYNC; 1082 } else { 1083 pipe->pipe_state &= ~PIPE_ASYNC; 1084 } 1085 mutex_exit(lock); 1086 return (0); 1087 1088 case FIONREAD: 1089 mutex_enter(lock); 1090 #ifndef PIPE_NODIRECT 1091 if (pipe->pipe_state & PIPE_DIRECTW) 1092 *(int *)data = pipe->pipe_map.cnt; 1093 else 1094 #endif 1095 *(int *)data = pipe->pipe_buffer.cnt; 1096 mutex_exit(lock); 1097 return (0); 1098 1099 case FIONWRITE: 1100 /* Look at other side */ 1101 pipe = pipe->pipe_peer; 1102 mutex_enter(lock); 1103 #ifndef PIPE_NODIRECT 1104 if (pipe->pipe_state & PIPE_DIRECTW) 1105 *(int *)data = pipe->pipe_map.cnt; 1106 else 1107 #endif 1108 *(int *)data = pipe->pipe_buffer.cnt; 1109 mutex_exit(lock); 1110 return (0); 1111 1112 case FIONSPACE: 1113 /* Look at other side */ 1114 pipe = pipe->pipe_peer; 1115 mutex_enter(lock); 1116 #ifndef PIPE_NODIRECT 1117 /* 1118 * If we're in direct-mode, we don't really have a 1119 * send queue, and any other write will block. Thus 1120 * zero seems like the best answer. 1121 */ 1122 if (pipe->pipe_state & PIPE_DIRECTW) 1123 *(int *)data = 0; 1124 else 1125 #endif 1126 *(int *)data = pipe->pipe_buffer.size - 1127 pipe->pipe_buffer.cnt; 1128 mutex_exit(lock); 1129 return (0); 1130 1131 case TIOCSPGRP: 1132 case FIOSETOWN: 1133 return fsetown(&pipe->pipe_pgid, cmd, data); 1134 1135 case TIOCGPGRP: 1136 case FIOGETOWN: 1137 return fgetown(pipe->pipe_pgid, cmd, data); 1138 1139 } 1140 return (EPASSTHROUGH); 1141 } 1142 1143 int 1144 pipe_poll(file_t *fp, int events) 1145 { 1146 struct pipe *rpipe = fp->f_data; 1147 struct pipe *wpipe; 1148 int eof = 0; 1149 int revents = 0; 1150 1151 mutex_enter(rpipe->pipe_lock); 1152 wpipe = rpipe->pipe_peer; 1153 1154 if (events & (POLLIN | POLLRDNORM)) 1155 if ((rpipe->pipe_buffer.cnt > 0) || 1156 #ifndef PIPE_NODIRECT 1157 (rpipe->pipe_state & PIPE_DIRECTR) || 1158 #endif 1159 (rpipe->pipe_state & PIPE_EOF)) 1160 revents |= events & (POLLIN | POLLRDNORM); 1161 1162 eof |= (rpipe->pipe_state & PIPE_EOF); 1163 1164 if (wpipe == NULL) 1165 revents |= events & (POLLOUT | POLLWRNORM); 1166 else { 1167 if (events & (POLLOUT | POLLWRNORM)) 1168 if ((wpipe->pipe_state & PIPE_EOF) || ( 1169 #ifndef PIPE_NODIRECT 1170 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 1171 #endif 1172 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 1173 revents |= events & (POLLOUT | POLLWRNORM); 1174 1175 eof |= (wpipe->pipe_state & PIPE_EOF); 1176 } 1177 1178 if (wpipe == NULL || eof) 1179 revents |= POLLHUP; 1180 1181 if (revents == 0) { 1182 if (events & (POLLIN | POLLRDNORM)) 1183 selrecord(curlwp, &rpipe->pipe_sel); 1184 1185 if (events & (POLLOUT | POLLWRNORM)) 1186 selrecord(curlwp, &wpipe->pipe_sel); 1187 } 1188 mutex_exit(rpipe->pipe_lock); 1189 1190 return (revents); 1191 } 1192 1193 static int 1194 pipe_stat(file_t *fp, struct stat *ub) 1195 { 1196 struct pipe *pipe = fp->f_data; 1197 1198 mutex_enter(pipe->pipe_lock); 1199 memset(ub, 0, sizeof(*ub)); 1200 ub->st_mode = S_IFIFO | S_IRUSR | S_IWUSR; 1201 ub->st_blksize = pipe->pipe_buffer.size; 1202 if (ub->st_blksize == 0 && pipe->pipe_peer) 1203 ub->st_blksize = pipe->pipe_peer->pipe_buffer.size; 1204 ub->st_size = pipe->pipe_buffer.cnt; 1205 ub->st_blocks = (ub->st_size) ? 1 : 0; 1206 ub->st_atimespec = pipe->pipe_atime; 1207 ub->st_mtimespec = pipe->pipe_mtime; 1208 ub->st_ctimespec = ub->st_birthtimespec = pipe->pipe_btime; 1209 ub->st_uid = kauth_cred_geteuid(fp->f_cred); 1210 ub->st_gid = kauth_cred_getegid(fp->f_cred); 1211 1212 /* 1213 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 1214 * XXX (st_dev, st_ino) should be unique. 1215 */ 1216 mutex_exit(pipe->pipe_lock); 1217 return 0; 1218 } 1219 1220 static int 1221 pipe_close(file_t *fp) 1222 { 1223 struct pipe *pipe = fp->f_data; 1224 1225 fp->f_data = NULL; 1226 pipeclose(pipe); 1227 return (0); 1228 } 1229 1230 static void 1231 pipe_restart(file_t *fp) 1232 { 1233 struct pipe *pipe = fp->f_data; 1234 1235 /* 1236 * Unblock blocked reads/writes in order to allow close() to complete. 1237 * System calls return ERESTART so that the fd is revalidated. 1238 * (Partial writes return the transfer length.) 1239 */ 1240 mutex_enter(pipe->pipe_lock); 1241 pipe->pipe_state |= PIPE_RESTART; 1242 /* Wakeup both cvs, maybe we only need one, but maybe there are some 1243 * other paths where wakeup is needed, and it saves deciding which! */ 1244 cv_broadcast(&pipe->pipe_rcv); 1245 cv_broadcast(&pipe->pipe_wcv); 1246 mutex_exit(pipe->pipe_lock); 1247 } 1248 1249 static void 1250 pipe_free_kmem(struct pipe *pipe) 1251 { 1252 1253 if (pipe->pipe_buffer.buffer != NULL) { 1254 if (pipe->pipe_buffer.size > PIPE_SIZE) { 1255 atomic_dec_uint(&nbigpipe); 1256 } 1257 if (pipe->pipe_buffer.buffer != (void *)pipe->pipe_kmem) { 1258 uvm_km_free(kernel_map, 1259 (vaddr_t)pipe->pipe_buffer.buffer, 1260 pipe->pipe_buffer.size, UVM_KMF_PAGEABLE); 1261 atomic_add_int(&amountpipekva, 1262 -pipe->pipe_buffer.size); 1263 } 1264 pipe->pipe_buffer.buffer = NULL; 1265 } 1266 #ifndef PIPE_NODIRECT 1267 if (pipe->pipe_map.kva != 0) { 1268 pipe_loan_free(pipe); 1269 pipe->pipe_map.cnt = 0; 1270 pipe->pipe_map.kva = 0; 1271 pipe->pipe_map.pos = 0; 1272 pipe->pipe_map.npages = 0; 1273 } 1274 #endif /* !PIPE_NODIRECT */ 1275 } 1276 1277 /* 1278 * Shutdown the pipe. 1279 */ 1280 static void 1281 pipeclose(struct pipe *pipe) 1282 { 1283 kmutex_t *lock; 1284 struct pipe *ppipe; 1285 1286 if (pipe == NULL) 1287 return; 1288 1289 KASSERT(cv_is_valid(&pipe->pipe_rcv)); 1290 KASSERT(cv_is_valid(&pipe->pipe_wcv)); 1291 KASSERT(cv_is_valid(&pipe->pipe_draincv)); 1292 KASSERT(cv_is_valid(&pipe->pipe_lkcv)); 1293 1294 lock = pipe->pipe_lock; 1295 if (lock == NULL) 1296 /* Must have failed during create */ 1297 goto free_resources; 1298 1299 mutex_enter(lock); 1300 pipeselwakeup(pipe, pipe, POLL_HUP); 1301 1302 /* 1303 * If the other side is blocked, wake it up saying that 1304 * we want to close it down. 1305 */ 1306 pipe->pipe_state |= PIPE_EOF; 1307 if (pipe->pipe_busy) { 1308 while (pipe->pipe_busy) { 1309 cv_broadcast(&pipe->pipe_wcv); 1310 cv_wait_sig(&pipe->pipe_draincv, lock); 1311 } 1312 } 1313 1314 /* 1315 * Disconnect from peer. 1316 */ 1317 if ((ppipe = pipe->pipe_peer) != NULL) { 1318 pipeselwakeup(ppipe, ppipe, POLL_HUP); 1319 ppipe->pipe_state |= PIPE_EOF; 1320 cv_broadcast(&ppipe->pipe_rcv); 1321 ppipe->pipe_peer = NULL; 1322 } 1323 1324 /* 1325 * Any knote objects still left in the list are 1326 * the one attached by peer. Since no one will 1327 * traverse this list, we just clear it. 1328 */ 1329 SLIST_INIT(&pipe->pipe_sel.sel_klist); 1330 1331 KASSERT((pipe->pipe_state & PIPE_LOCKFL) == 0); 1332 mutex_exit(lock); 1333 mutex_obj_free(lock); 1334 1335 /* 1336 * Free resources. 1337 */ 1338 free_resources: 1339 pipe->pipe_pgid = 0; 1340 pipe->pipe_state = PIPE_SIGNALR; 1341 pipe_free_kmem(pipe); 1342 if (pipe->pipe_kmem != 0) { 1343 pool_cache_put(pipe_rd_cache, pipe); 1344 } else { 1345 pool_cache_put(pipe_wr_cache, pipe); 1346 } 1347 } 1348 1349 static void 1350 filt_pipedetach(struct knote *kn) 1351 { 1352 struct pipe *pipe; 1353 kmutex_t *lock; 1354 1355 pipe = ((file_t *)kn->kn_obj)->f_data; 1356 lock = pipe->pipe_lock; 1357 1358 mutex_enter(lock); 1359 1360 switch(kn->kn_filter) { 1361 case EVFILT_WRITE: 1362 /* Need the peer structure, not our own. */ 1363 pipe = pipe->pipe_peer; 1364 1365 /* If reader end already closed, just return. */ 1366 if (pipe == NULL) { 1367 mutex_exit(lock); 1368 return; 1369 } 1370 1371 break; 1372 default: 1373 /* Nothing to do. */ 1374 break; 1375 } 1376 1377 KASSERT(kn->kn_hook == pipe); 1378 SLIST_REMOVE(&pipe->pipe_sel.sel_klist, kn, knote, kn_selnext); 1379 mutex_exit(lock); 1380 } 1381 1382 static int 1383 filt_piperead(struct knote *kn, long hint) 1384 { 1385 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_data; 1386 struct pipe *wpipe; 1387 1388 if ((hint & NOTE_SUBMIT) == 0) { 1389 mutex_enter(rpipe->pipe_lock); 1390 } 1391 wpipe = rpipe->pipe_peer; 1392 kn->kn_data = rpipe->pipe_buffer.cnt; 1393 1394 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1395 kn->kn_data = rpipe->pipe_map.cnt; 1396 1397 if ((rpipe->pipe_state & PIPE_EOF) || 1398 (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1399 kn->kn_flags |= EV_EOF; 1400 if ((hint & NOTE_SUBMIT) == 0) { 1401 mutex_exit(rpipe->pipe_lock); 1402 } 1403 return (1); 1404 } 1405 1406 if ((hint & NOTE_SUBMIT) == 0) { 1407 mutex_exit(rpipe->pipe_lock); 1408 } 1409 return (kn->kn_data > 0); 1410 } 1411 1412 static int 1413 filt_pipewrite(struct knote *kn, long hint) 1414 { 1415 struct pipe *rpipe = ((file_t *)kn->kn_obj)->f_data; 1416 struct pipe *wpipe; 1417 1418 if ((hint & NOTE_SUBMIT) == 0) { 1419 mutex_enter(rpipe->pipe_lock); 1420 } 1421 wpipe = rpipe->pipe_peer; 1422 1423 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1424 kn->kn_data = 0; 1425 kn->kn_flags |= EV_EOF; 1426 if ((hint & NOTE_SUBMIT) == 0) { 1427 mutex_exit(rpipe->pipe_lock); 1428 } 1429 return (1); 1430 } 1431 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1432 if (wpipe->pipe_state & PIPE_DIRECTW) 1433 kn->kn_data = 0; 1434 1435 if ((hint & NOTE_SUBMIT) == 0) { 1436 mutex_exit(rpipe->pipe_lock); 1437 } 1438 return (kn->kn_data >= PIPE_BUF); 1439 } 1440 1441 static const struct filterops pipe_rfiltops = 1442 { 1, NULL, filt_pipedetach, filt_piperead }; 1443 static const struct filterops pipe_wfiltops = 1444 { 1, NULL, filt_pipedetach, filt_pipewrite }; 1445 1446 static int 1447 pipe_kqfilter(file_t *fp, struct knote *kn) 1448 { 1449 struct pipe *pipe; 1450 kmutex_t *lock; 1451 1452 pipe = ((file_t *)kn->kn_obj)->f_data; 1453 lock = pipe->pipe_lock; 1454 1455 mutex_enter(lock); 1456 1457 switch (kn->kn_filter) { 1458 case EVFILT_READ: 1459 kn->kn_fop = &pipe_rfiltops; 1460 break; 1461 case EVFILT_WRITE: 1462 kn->kn_fop = &pipe_wfiltops; 1463 pipe = pipe->pipe_peer; 1464 if (pipe == NULL) { 1465 /* Other end of pipe has been closed. */ 1466 mutex_exit(lock); 1467 return (EBADF); 1468 } 1469 break; 1470 default: 1471 mutex_exit(lock); 1472 return (EINVAL); 1473 } 1474 1475 kn->kn_hook = pipe; 1476 SLIST_INSERT_HEAD(&pipe->pipe_sel.sel_klist, kn, kn_selnext); 1477 mutex_exit(lock); 1478 1479 return (0); 1480 } 1481 1482 /* 1483 * Handle pipe sysctls. 1484 */ 1485 SYSCTL_SETUP(sysctl_kern_pipe_setup, "sysctl kern.pipe subtree setup") 1486 { 1487 1488 sysctl_createv(clog, 0, NULL, NULL, 1489 CTLFLAG_PERMANENT, 1490 CTLTYPE_NODE, "kern", NULL, 1491 NULL, 0, NULL, 0, 1492 CTL_KERN, CTL_EOL); 1493 sysctl_createv(clog, 0, NULL, NULL, 1494 CTLFLAG_PERMANENT, 1495 CTLTYPE_NODE, "pipe", 1496 SYSCTL_DESCR("Pipe settings"), 1497 NULL, 0, NULL, 0, 1498 CTL_KERN, KERN_PIPE, CTL_EOL); 1499 1500 sysctl_createv(clog, 0, NULL, NULL, 1501 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1502 CTLTYPE_INT, "maxkvasz", 1503 SYSCTL_DESCR("Maximum amount of kernel memory to be " 1504 "used for pipes"), 1505 NULL, 0, &maxpipekva, 0, 1506 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXKVASZ, CTL_EOL); 1507 sysctl_createv(clog, 0, NULL, NULL, 1508 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1509 CTLTYPE_INT, "maxloankvasz", 1510 SYSCTL_DESCR("Limit for direct transfers via page loan"), 1511 NULL, 0, &limitpipekva, 0, 1512 CTL_KERN, KERN_PIPE, KERN_PIPE_LIMITKVA, CTL_EOL); 1513 sysctl_createv(clog, 0, NULL, NULL, 1514 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1515 CTLTYPE_INT, "maxbigpipes", 1516 SYSCTL_DESCR("Maximum number of \"big\" pipes"), 1517 NULL, 0, &maxbigpipes, 0, 1518 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXBIGPIPES, CTL_EOL); 1519 sysctl_createv(clog, 0, NULL, NULL, 1520 CTLFLAG_PERMANENT, 1521 CTLTYPE_INT, "nbigpipes", 1522 SYSCTL_DESCR("Number of \"big\" pipes"), 1523 NULL, 0, &nbigpipe, 0, 1524 CTL_KERN, KERN_PIPE, KERN_PIPE_NBIGPIPES, CTL_EOL); 1525 sysctl_createv(clog, 0, NULL, NULL, 1526 CTLFLAG_PERMANENT, 1527 CTLTYPE_INT, "kvasize", 1528 SYSCTL_DESCR("Amount of kernel memory consumed by pipe " 1529 "buffers"), 1530 NULL, 0, &amountpipekva, 0, 1531 CTL_KERN, KERN_PIPE, KERN_PIPE_KVASIZE, CTL_EOL); 1532 } 1533