1 /* $NetBSD: sys_pipe.c,v 1.76 2006/10/12 01:32:18 christos Exp $ */ 2 3 /*- 4 * Copyright (c) 2003 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Paul Kranenburg. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Copyright (c) 1996 John S. Dyson 41 * All rights reserved. 42 * 43 * Redistribution and use in source and binary forms, with or without 44 * modification, are permitted provided that the following conditions 45 * are met: 46 * 1. Redistributions of source code must retain the above copyright 47 * notice immediately at the beginning of the file, without modification, 48 * this list of conditions, and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. Absolutely no warranty of function or purpose is made by the author 53 * John S. Dyson. 54 * 4. Modifications may be freely made to this file if the above conditions 55 * are met. 56 * 57 * $FreeBSD: src/sys/kern/sys_pipe.c,v 1.95 2002/03/09 22:06:31 alfred Exp $ 58 */ 59 60 /* 61 * This file contains a high-performance replacement for the socket-based 62 * pipes scheme originally used in FreeBSD/4.4Lite. It does not support 63 * all features of sockets, but does do everything that pipes normally 64 * do. 65 * 66 * Adaption for NetBSD UVM, including uvm_loan() based direct write, was 67 * written by Jaromir Dolecek. 68 */ 69 70 /* 71 * This code has two modes of operation, a small write mode and a large 72 * write mode. The small write mode acts like conventional pipes with 73 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 74 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 75 * and PIPE_SIZE in size it is mapped read-only into the kernel address space 76 * using the UVM page loan facility from where the receiving process can copy 77 * the data directly from the pages in the sending process. 78 * 79 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 80 * happen for small transfers so that the system will not spend all of 81 * its time context switching. PIPE_SIZE is constrained by the 82 * amount of kernel virtual memory. 83 */ 84 85 #include <sys/cdefs.h> 86 __KERNEL_RCSID(0, "$NetBSD: sys_pipe.c,v 1.76 2006/10/12 01:32:18 christos Exp $"); 87 88 #include <sys/param.h> 89 #include <sys/systm.h> 90 #include <sys/proc.h> 91 #include <sys/fcntl.h> 92 #include <sys/file.h> 93 #include <sys/filedesc.h> 94 #include <sys/filio.h> 95 #include <sys/kernel.h> 96 #include <sys/ttycom.h> 97 #include <sys/stat.h> 98 #include <sys/malloc.h> 99 #include <sys/poll.h> 100 #include <sys/signalvar.h> 101 #include <sys/vnode.h> 102 #include <sys/uio.h> 103 #include <sys/lock.h> 104 #include <sys/select.h> 105 #include <sys/mount.h> 106 #include <sys/sa.h> 107 #include <sys/syscallargs.h> 108 #include <uvm/uvm.h> 109 #include <sys/sysctl.h> 110 #include <sys/kernel.h> 111 #include <sys/kauth.h> 112 113 #include <sys/pipe.h> 114 115 /* 116 * Use this define if you want to disable *fancy* VM things. Expect an 117 * approx 30% decrease in transfer rate. 118 */ 119 /* #define PIPE_NODIRECT */ 120 121 /* 122 * interfaces to the outside world 123 */ 124 static int pipe_read(struct file *fp, off_t *offset, struct uio *uio, 125 kauth_cred_t cred, int flags); 126 static int pipe_write(struct file *fp, off_t *offset, struct uio *uio, 127 kauth_cred_t cred, int flags); 128 static int pipe_close(struct file *fp, struct lwp *l); 129 static int pipe_poll(struct file *fp, int events, struct lwp *l); 130 static int pipe_kqfilter(struct file *fp, struct knote *kn); 131 static int pipe_stat(struct file *fp, struct stat *sb, struct lwp *l); 132 static int pipe_ioctl(struct file *fp, u_long cmd, void *data, 133 struct lwp *l); 134 135 static const struct fileops pipeops = { 136 pipe_read, pipe_write, pipe_ioctl, fnullop_fcntl, pipe_poll, 137 pipe_stat, pipe_close, pipe_kqfilter 138 }; 139 140 /* 141 * Default pipe buffer size(s), this can be kind-of large now because pipe 142 * space is pageable. The pipe code will try to maintain locality of 143 * reference for performance reasons, so small amounts of outstanding I/O 144 * will not wipe the cache. 145 */ 146 #define MINPIPESIZE (PIPE_SIZE/3) 147 #define MAXPIPESIZE (2*PIPE_SIZE/3) 148 149 /* 150 * Maximum amount of kva for pipes -- this is kind-of a soft limit, but 151 * is there so that on large systems, we don't exhaust it. 152 */ 153 #define MAXPIPEKVA (8*1024*1024) 154 static int maxpipekva = MAXPIPEKVA; 155 156 /* 157 * Limit for direct transfers, we cannot, of course limit 158 * the amount of kva for pipes in general though. 159 */ 160 #define LIMITPIPEKVA (16*1024*1024) 161 static int limitpipekva = LIMITPIPEKVA; 162 163 /* 164 * Limit the number of "big" pipes 165 */ 166 #define LIMITBIGPIPES 32 167 static int maxbigpipes = LIMITBIGPIPES; 168 static int nbigpipe = 0; 169 170 /* 171 * Amount of KVA consumed by pipe buffers. 172 */ 173 static int amountpipekva = 0; 174 175 MALLOC_DEFINE(M_PIPE, "pipe", "Pipe structures"); 176 177 static void pipeclose(struct file *fp, struct pipe *pipe); 178 static void pipe_free_kmem(struct pipe *pipe); 179 static int pipe_create(struct pipe **pipep, int allockva); 180 static int pipelock(struct pipe *pipe, int catch); 181 static inline void pipeunlock(struct pipe *pipe); 182 static void pipeselwakeup(struct pipe *pipe, struct pipe *sigp, int code); 183 #ifndef PIPE_NODIRECT 184 static int pipe_direct_write(struct file *fp, struct pipe *wpipe, 185 struct uio *uio); 186 #endif 187 static int pipespace(struct pipe *pipe, int size); 188 189 #ifndef PIPE_NODIRECT 190 static int pipe_loan_alloc(struct pipe *, int); 191 static void pipe_loan_free(struct pipe *); 192 #endif /* PIPE_NODIRECT */ 193 194 static POOL_INIT(pipe_pool, sizeof(struct pipe), 0, 0, 0, "pipepl", 195 &pool_allocator_nointr); 196 197 /* 198 * The pipe system call for the DTYPE_PIPE type of pipes 199 */ 200 201 /* ARGSUSED */ 202 int 203 sys_pipe(struct lwp *l, void *v __unused, register_t *retval) 204 { 205 struct file *rf, *wf; 206 struct pipe *rpipe, *wpipe; 207 int fd, error; 208 209 rpipe = wpipe = NULL; 210 if (pipe_create(&rpipe, 1) || pipe_create(&wpipe, 0)) { 211 pipeclose(NULL, rpipe); 212 pipeclose(NULL, wpipe); 213 return (ENFILE); 214 } 215 216 /* 217 * Note: the file structure returned from falloc() is marked 218 * as 'larval' initially. Unless we mark it as 'mature' by 219 * FILE_SET_MATURE(), any attempt to do anything with it would 220 * return EBADF, including e.g. dup(2) or close(2). This avoids 221 * file descriptor races if we block in the second falloc(). 222 */ 223 224 error = falloc(l, &rf, &fd); 225 if (error) 226 goto free2; 227 retval[0] = fd; 228 rf->f_flag = FREAD; 229 rf->f_type = DTYPE_PIPE; 230 rf->f_data = (caddr_t)rpipe; 231 rf->f_ops = &pipeops; 232 233 error = falloc(l, &wf, &fd); 234 if (error) 235 goto free3; 236 retval[1] = fd; 237 wf->f_flag = FWRITE; 238 wf->f_type = DTYPE_PIPE; 239 wf->f_data = (caddr_t)wpipe; 240 wf->f_ops = &pipeops; 241 242 rpipe->pipe_peer = wpipe; 243 wpipe->pipe_peer = rpipe; 244 245 FILE_SET_MATURE(rf); 246 FILE_SET_MATURE(wf); 247 FILE_UNUSE(rf, l); 248 FILE_UNUSE(wf, l); 249 return (0); 250 free3: 251 FILE_UNUSE(rf, l); 252 ffree(rf); 253 fdremove(l->l_proc->p_fd, retval[0]); 254 free2: 255 pipeclose(NULL, wpipe); 256 pipeclose(NULL, rpipe); 257 258 return (error); 259 } 260 261 /* 262 * Allocate kva for pipe circular buffer, the space is pageable 263 * This routine will 'realloc' the size of a pipe safely, if it fails 264 * it will retain the old buffer. 265 * If it fails it will return ENOMEM. 266 */ 267 static int 268 pipespace(struct pipe *pipe, int size) 269 { 270 caddr_t buffer; 271 /* 272 * Allocate pageable virtual address space. Physical memory is 273 * allocated on demand. 274 */ 275 buffer = (caddr_t) uvm_km_alloc(kernel_map, round_page(size), 0, 276 UVM_KMF_PAGEABLE); 277 if (buffer == NULL) 278 return (ENOMEM); 279 280 /* free old resources if we're resizing */ 281 pipe_free_kmem(pipe); 282 pipe->pipe_buffer.buffer = buffer; 283 pipe->pipe_buffer.size = size; 284 pipe->pipe_buffer.in = 0; 285 pipe->pipe_buffer.out = 0; 286 pipe->pipe_buffer.cnt = 0; 287 amountpipekva += pipe->pipe_buffer.size; 288 return (0); 289 } 290 291 /* 292 * Initialize and allocate VM and memory for pipe. 293 */ 294 static int 295 pipe_create(struct pipe **pipep, int allockva) 296 { 297 struct pipe *pipe; 298 int error; 299 300 pipe = *pipep = pool_get(&pipe_pool, PR_WAITOK); 301 302 /* Initialize */ 303 memset(pipe, 0, sizeof(struct pipe)); 304 pipe->pipe_state = PIPE_SIGNALR; 305 306 getmicrotime(&pipe->pipe_ctime); 307 pipe->pipe_atime = pipe->pipe_ctime; 308 pipe->pipe_mtime = pipe->pipe_ctime; 309 simple_lock_init(&pipe->pipe_slock); 310 311 if (allockva && (error = pipespace(pipe, PIPE_SIZE))) 312 return (error); 313 314 return (0); 315 } 316 317 318 /* 319 * Lock a pipe for I/O, blocking other access 320 * Called with pipe spin lock held. 321 * Return with pipe spin lock released on success. 322 */ 323 static int 324 pipelock(struct pipe *pipe, int catch) 325 { 326 327 LOCK_ASSERT(simple_lock_held(&pipe->pipe_slock)); 328 329 while (pipe->pipe_state & PIPE_LOCKFL) { 330 int error; 331 const int pcatch = catch ? PCATCH : 0; 332 333 pipe->pipe_state |= PIPE_LWANT; 334 error = ltsleep(pipe, PSOCK | pcatch, "pipelk", 0, 335 &pipe->pipe_slock); 336 if (error != 0) 337 return error; 338 } 339 340 pipe->pipe_state |= PIPE_LOCKFL; 341 simple_unlock(&pipe->pipe_slock); 342 343 return 0; 344 } 345 346 /* 347 * unlock a pipe I/O lock 348 */ 349 static inline void 350 pipeunlock(struct pipe *pipe) 351 { 352 353 KASSERT(pipe->pipe_state & PIPE_LOCKFL); 354 355 pipe->pipe_state &= ~PIPE_LOCKFL; 356 if (pipe->pipe_state & PIPE_LWANT) { 357 pipe->pipe_state &= ~PIPE_LWANT; 358 wakeup(pipe); 359 } 360 } 361 362 /* 363 * Select/poll wakup. This also sends SIGIO to peer connected to 364 * 'sigpipe' side of pipe. 365 */ 366 static void 367 pipeselwakeup(struct pipe *selp, struct pipe *sigp, int code) 368 { 369 int band; 370 371 selnotify(&selp->pipe_sel, NOTE_SUBMIT); 372 373 if (sigp == NULL || (sigp->pipe_state & PIPE_ASYNC) == 0) 374 return; 375 376 switch (code) { 377 case POLL_IN: 378 band = POLLIN|POLLRDNORM; 379 break; 380 case POLL_OUT: 381 band = POLLOUT|POLLWRNORM; 382 break; 383 case POLL_HUP: 384 band = POLLHUP; 385 break; 386 #if POLL_HUP != POLL_ERR 387 case POLL_ERR: 388 band = POLLERR; 389 break; 390 #endif 391 default: 392 band = 0; 393 #ifdef DIAGNOSTIC 394 printf("bad siginfo code %d in pipe notification.\n", code); 395 #endif 396 break; 397 } 398 399 fownsignal(sigp->pipe_pgid, SIGIO, code, band, selp); 400 } 401 402 /* ARGSUSED */ 403 static int 404 pipe_read(struct file *fp, off_t *offset __unused, struct uio *uio, 405 kauth_cred_t cred __unused, int flags __unused) 406 { 407 struct pipe *rpipe = (struct pipe *) fp->f_data; 408 struct pipebuf *bp = &rpipe->pipe_buffer; 409 int error; 410 size_t nread = 0; 411 size_t size; 412 size_t ocnt; 413 414 PIPE_LOCK(rpipe); 415 ++rpipe->pipe_busy; 416 ocnt = bp->cnt; 417 418 again: 419 error = pipelock(rpipe, 1); 420 if (error) 421 goto unlocked_error; 422 423 while (uio->uio_resid) { 424 /* 425 * normal pipe buffer receive 426 */ 427 if (bp->cnt > 0) { 428 size = bp->size - bp->out; 429 if (size > bp->cnt) 430 size = bp->cnt; 431 if (size > uio->uio_resid) 432 size = uio->uio_resid; 433 434 error = uiomove(&bp->buffer[bp->out], size, uio); 435 if (error) 436 break; 437 438 bp->out += size; 439 if (bp->out >= bp->size) 440 bp->out = 0; 441 442 bp->cnt -= size; 443 444 /* 445 * If there is no more to read in the pipe, reset 446 * its pointers to the beginning. This improves 447 * cache hit stats. 448 */ 449 if (bp->cnt == 0) { 450 bp->in = 0; 451 bp->out = 0; 452 } 453 nread += size; 454 #ifndef PIPE_NODIRECT 455 } else if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) { 456 /* 457 * Direct copy, bypassing a kernel buffer. 458 */ 459 caddr_t va; 460 461 KASSERT(rpipe->pipe_state & PIPE_DIRECTW); 462 463 size = rpipe->pipe_map.cnt; 464 if (size > uio->uio_resid) 465 size = uio->uio_resid; 466 467 va = (caddr_t) rpipe->pipe_map.kva + 468 rpipe->pipe_map.pos; 469 error = uiomove(va, size, uio); 470 if (error) 471 break; 472 nread += size; 473 rpipe->pipe_map.pos += size; 474 rpipe->pipe_map.cnt -= size; 475 if (rpipe->pipe_map.cnt == 0) { 476 PIPE_LOCK(rpipe); 477 rpipe->pipe_state &= ~PIPE_DIRECTR; 478 wakeup(rpipe); 479 PIPE_UNLOCK(rpipe); 480 } 481 #endif 482 } else { 483 /* 484 * Break if some data was read. 485 */ 486 if (nread > 0) 487 break; 488 489 PIPE_LOCK(rpipe); 490 491 /* 492 * detect EOF condition 493 * read returns 0 on EOF, no need to set error 494 */ 495 if (rpipe->pipe_state & PIPE_EOF) { 496 PIPE_UNLOCK(rpipe); 497 break; 498 } 499 500 /* 501 * don't block on non-blocking I/O 502 */ 503 if (fp->f_flag & FNONBLOCK) { 504 PIPE_UNLOCK(rpipe); 505 error = EAGAIN; 506 break; 507 } 508 509 /* 510 * Unlock the pipe buffer for our remaining processing. 511 * We will either break out with an error or we will 512 * sleep and relock to loop. 513 */ 514 pipeunlock(rpipe); 515 516 /* 517 * The PIPE_DIRECTR flag is not under the control 518 * of the long-term lock (see pipe_direct_write()), 519 * so re-check now while holding the spin lock. 520 */ 521 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) 522 goto again; 523 524 /* 525 * We want to read more, wake up select/poll. 526 */ 527 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_IN); 528 529 /* 530 * If the "write-side" is blocked, wake it up now. 531 */ 532 if (rpipe->pipe_state & PIPE_WANTW) { 533 rpipe->pipe_state &= ~PIPE_WANTW; 534 wakeup(rpipe); 535 } 536 537 /* Now wait until the pipe is filled */ 538 rpipe->pipe_state |= PIPE_WANTR; 539 error = ltsleep(rpipe, PSOCK | PCATCH, 540 "piperd", 0, &rpipe->pipe_slock); 541 if (error != 0) 542 goto unlocked_error; 543 goto again; 544 } 545 } 546 547 if (error == 0) 548 getmicrotime(&rpipe->pipe_atime); 549 550 PIPE_LOCK(rpipe); 551 pipeunlock(rpipe); 552 553 unlocked_error: 554 --rpipe->pipe_busy; 555 556 /* 557 * PIPE_WANTCLOSE processing only makes sense if pipe_busy is 0. 558 */ 559 if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANTCLOSE)) { 560 rpipe->pipe_state &= ~(PIPE_WANTCLOSE|PIPE_WANTW); 561 wakeup(rpipe); 562 } else if (bp->cnt < MINPIPESIZE) { 563 /* 564 * Handle write blocking hysteresis. 565 */ 566 if (rpipe->pipe_state & PIPE_WANTW) { 567 rpipe->pipe_state &= ~PIPE_WANTW; 568 wakeup(rpipe); 569 } 570 } 571 572 /* 573 * If anything was read off the buffer, signal to the writer it's 574 * possible to write more data. Also send signal if we are here for the 575 * first time after last write. 576 */ 577 if ((bp->size - bp->cnt) >= PIPE_BUF 578 && (ocnt != bp->cnt || (rpipe->pipe_state & PIPE_SIGNALR))) { 579 pipeselwakeup(rpipe, rpipe->pipe_peer, POLL_OUT); 580 rpipe->pipe_state &= ~PIPE_SIGNALR; 581 } 582 583 PIPE_UNLOCK(rpipe); 584 return (error); 585 } 586 587 #ifndef PIPE_NODIRECT 588 /* 589 * Allocate structure for loan transfer. 590 */ 591 static int 592 pipe_loan_alloc(struct pipe *wpipe, int npages) 593 { 594 vsize_t len; 595 596 len = (vsize_t)npages << PAGE_SHIFT; 597 wpipe->pipe_map.kva = uvm_km_alloc(kernel_map, len, 0, 598 UVM_KMF_VAONLY | UVM_KMF_WAITVA); 599 if (wpipe->pipe_map.kva == 0) 600 return (ENOMEM); 601 602 amountpipekva += len; 603 wpipe->pipe_map.npages = npages; 604 wpipe->pipe_map.pgs = malloc(npages * sizeof(struct vm_page *), M_PIPE, 605 M_WAITOK); 606 return (0); 607 } 608 609 /* 610 * Free resources allocated for loan transfer. 611 */ 612 static void 613 pipe_loan_free(struct pipe *wpipe) 614 { 615 vsize_t len; 616 617 len = (vsize_t)wpipe->pipe_map.npages << PAGE_SHIFT; 618 uvm_km_free(kernel_map, wpipe->pipe_map.kva, len, UVM_KMF_VAONLY); 619 wpipe->pipe_map.kva = 0; 620 amountpipekva -= len; 621 free(wpipe->pipe_map.pgs, M_PIPE); 622 wpipe->pipe_map.pgs = NULL; 623 } 624 625 /* 626 * NetBSD direct write, using uvm_loan() mechanism. 627 * This implements the pipe buffer write mechanism. Note that only 628 * a direct write OR a normal pipe write can be pending at any given time. 629 * If there are any characters in the pipe buffer, the direct write will 630 * be deferred until the receiving process grabs all of the bytes from 631 * the pipe buffer. Then the direct mapping write is set-up. 632 * 633 * Called with the long-term pipe lock held. 634 */ 635 static int 636 pipe_direct_write(struct file *fp __unused, struct pipe *wpipe, 637 struct uio *uio) 638 { 639 int error, npages, j; 640 struct vm_page **pgs; 641 vaddr_t bbase, kva, base, bend; 642 vsize_t blen, bcnt; 643 voff_t bpos; 644 645 KASSERT(wpipe->pipe_map.cnt == 0); 646 647 /* 648 * Handle first PIPE_CHUNK_SIZE bytes of buffer. Deal with buffers 649 * not aligned to PAGE_SIZE. 650 */ 651 bbase = (vaddr_t)uio->uio_iov->iov_base; 652 base = trunc_page(bbase); 653 bend = round_page(bbase + uio->uio_iov->iov_len); 654 blen = bend - base; 655 bpos = bbase - base; 656 657 if (blen > PIPE_DIRECT_CHUNK) { 658 blen = PIPE_DIRECT_CHUNK; 659 bend = base + blen; 660 bcnt = PIPE_DIRECT_CHUNK - bpos; 661 } else { 662 bcnt = uio->uio_iov->iov_len; 663 } 664 npages = blen >> PAGE_SHIFT; 665 666 /* 667 * Free the old kva if we need more pages than we have 668 * allocated. 669 */ 670 if (wpipe->pipe_map.kva != 0 && npages > wpipe->pipe_map.npages) 671 pipe_loan_free(wpipe); 672 673 /* Allocate new kva. */ 674 if (wpipe->pipe_map.kva == 0) { 675 error = pipe_loan_alloc(wpipe, npages); 676 if (error) 677 return (error); 678 } 679 680 /* Loan the write buffer memory from writer process */ 681 pgs = wpipe->pipe_map.pgs; 682 error = uvm_loan(&uio->uio_vmspace->vm_map, base, blen, 683 pgs, UVM_LOAN_TOPAGE); 684 if (error) { 685 pipe_loan_free(wpipe); 686 return (ENOMEM); /* so that caller fallback to ordinary write */ 687 } 688 689 /* Enter the loaned pages to kva */ 690 kva = wpipe->pipe_map.kva; 691 for (j = 0; j < npages; j++, kva += PAGE_SIZE) { 692 pmap_kenter_pa(kva, VM_PAGE_TO_PHYS(pgs[j]), VM_PROT_READ); 693 } 694 pmap_update(pmap_kernel()); 695 696 /* Now we can put the pipe in direct write mode */ 697 wpipe->pipe_map.pos = bpos; 698 wpipe->pipe_map.cnt = bcnt; 699 wpipe->pipe_state |= PIPE_DIRECTW; 700 701 /* 702 * But before we can let someone do a direct read, 703 * we have to wait until the pipe is drained. 704 */ 705 706 /* Relase the pipe lock while we wait */ 707 PIPE_LOCK(wpipe); 708 pipeunlock(wpipe); 709 710 while (error == 0 && wpipe->pipe_buffer.cnt > 0) { 711 if (wpipe->pipe_state & PIPE_WANTR) { 712 wpipe->pipe_state &= ~PIPE_WANTR; 713 wakeup(wpipe); 714 } 715 716 wpipe->pipe_state |= PIPE_WANTW; 717 error = ltsleep(wpipe, PSOCK | PCATCH, "pipdwc", 0, 718 &wpipe->pipe_slock); 719 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 720 error = EPIPE; 721 } 722 723 /* Pipe is drained; next read will off the direct buffer */ 724 wpipe->pipe_state |= PIPE_DIRECTR; 725 726 /* Wait until the reader is done */ 727 while (error == 0 && (wpipe->pipe_state & PIPE_DIRECTR)) { 728 if (wpipe->pipe_state & PIPE_WANTR) { 729 wpipe->pipe_state &= ~PIPE_WANTR; 730 wakeup(wpipe); 731 } 732 pipeselwakeup(wpipe, wpipe, POLL_IN); 733 error = ltsleep(wpipe, PSOCK | PCATCH, "pipdwt", 0, 734 &wpipe->pipe_slock); 735 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 736 error = EPIPE; 737 } 738 739 /* Take pipe out of direct write mode */ 740 wpipe->pipe_state &= ~(PIPE_DIRECTW | PIPE_DIRECTR); 741 742 /* Acquire the pipe lock and cleanup */ 743 (void)pipelock(wpipe, 0); 744 if (pgs != NULL) { 745 pmap_kremove(wpipe->pipe_map.kva, blen); 746 uvm_unloan(pgs, npages, UVM_LOAN_TOPAGE); 747 } 748 if (error || amountpipekva > maxpipekva) 749 pipe_loan_free(wpipe); 750 751 if (error) { 752 pipeselwakeup(wpipe, wpipe, POLL_ERR); 753 754 /* 755 * If nothing was read from what we offered, return error 756 * straight on. Otherwise update uio resid first. Caller 757 * will deal with the error condition, returning short 758 * write, error, or restarting the write(2) as appropriate. 759 */ 760 if (wpipe->pipe_map.cnt == bcnt) { 761 wpipe->pipe_map.cnt = 0; 762 wakeup(wpipe); 763 return (error); 764 } 765 766 bcnt -= wpipe->pipe_map.cnt; 767 } 768 769 uio->uio_resid -= bcnt; 770 /* uio_offset not updated, not set/used for write(2) */ 771 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + bcnt; 772 uio->uio_iov->iov_len -= bcnt; 773 if (uio->uio_iov->iov_len == 0) { 774 uio->uio_iov++; 775 uio->uio_iovcnt--; 776 } 777 778 wpipe->pipe_map.cnt = 0; 779 return (error); 780 } 781 #endif /* !PIPE_NODIRECT */ 782 783 static int 784 pipe_write(struct file *fp, off_t *offset __unused, struct uio *uio, 785 kauth_cred_t cred __unused, int flags __unused) 786 { 787 struct pipe *wpipe, *rpipe; 788 struct pipebuf *bp; 789 int error; 790 791 /* We want to write to our peer */ 792 rpipe = (struct pipe *) fp->f_data; 793 794 retry: 795 error = 0; 796 PIPE_LOCK(rpipe); 797 wpipe = rpipe->pipe_peer; 798 799 /* 800 * Detect loss of pipe read side, issue SIGPIPE if lost. 801 */ 802 if (wpipe == NULL) 803 error = EPIPE; 804 else if (simple_lock_try(&wpipe->pipe_slock) == 0) { 805 /* Deal with race for peer */ 806 PIPE_UNLOCK(rpipe); 807 goto retry; 808 } else if ((wpipe->pipe_state & PIPE_EOF) != 0) { 809 PIPE_UNLOCK(wpipe); 810 error = EPIPE; 811 } 812 813 PIPE_UNLOCK(rpipe); 814 if (error != 0) 815 return (error); 816 817 ++wpipe->pipe_busy; 818 819 /* Aquire the long-term pipe lock */ 820 if ((error = pipelock(wpipe,1)) != 0) { 821 --wpipe->pipe_busy; 822 if (wpipe->pipe_busy == 0 823 && (wpipe->pipe_state & PIPE_WANTCLOSE)) { 824 wpipe->pipe_state &= ~(PIPE_WANTCLOSE | PIPE_WANTR); 825 wakeup(wpipe); 826 } 827 PIPE_UNLOCK(wpipe); 828 return (error); 829 } 830 831 bp = &wpipe->pipe_buffer; 832 833 /* 834 * If it is advantageous to resize the pipe buffer, do so. 835 */ 836 if ((uio->uio_resid > PIPE_SIZE) && 837 (nbigpipe < maxbigpipes) && 838 #ifndef PIPE_NODIRECT 839 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 840 #endif 841 (bp->size <= PIPE_SIZE) && (bp->cnt == 0)) { 842 843 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) 844 nbigpipe++; 845 } 846 847 while (uio->uio_resid) { 848 size_t space; 849 850 #ifndef PIPE_NODIRECT 851 /* 852 * Pipe buffered writes cannot be coincidental with 853 * direct writes. Also, only one direct write can be 854 * in progress at any one time. We wait until the currently 855 * executing direct write is completed before continuing. 856 * 857 * We break out if a signal occurs or the reader goes away. 858 */ 859 while (error == 0 && wpipe->pipe_state & PIPE_DIRECTW) { 860 PIPE_LOCK(wpipe); 861 if (wpipe->pipe_state & PIPE_WANTR) { 862 wpipe->pipe_state &= ~PIPE_WANTR; 863 wakeup(wpipe); 864 } 865 pipeunlock(wpipe); 866 error = ltsleep(wpipe, PSOCK | PCATCH, 867 "pipbww", 0, &wpipe->pipe_slock); 868 869 (void)pipelock(wpipe, 0); 870 if (wpipe->pipe_state & PIPE_EOF) 871 error = EPIPE; 872 } 873 if (error) 874 break; 875 876 /* 877 * If the transfer is large, we can gain performance if 878 * we do process-to-process copies directly. 879 * If the write is non-blocking, we don't use the 880 * direct write mechanism. 881 * 882 * The direct write mechanism will detect the reader going 883 * away on us. 884 */ 885 if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) && 886 (fp->f_flag & FNONBLOCK) == 0 && 887 (wpipe->pipe_map.kva || (amountpipekva < limitpipekva))) { 888 error = pipe_direct_write(fp, wpipe, uio); 889 890 /* 891 * Break out if error occurred, unless it's ENOMEM. 892 * ENOMEM means we failed to allocate some resources 893 * for direct write, so we just fallback to ordinary 894 * write. If the direct write was successful, 895 * process rest of data via ordinary write. 896 */ 897 if (error == 0) 898 continue; 899 900 if (error != ENOMEM) 901 break; 902 } 903 #endif /* PIPE_NODIRECT */ 904 905 space = bp->size - bp->cnt; 906 907 /* Writes of size <= PIPE_BUF must be atomic. */ 908 if ((space < uio->uio_resid) && (uio->uio_resid <= PIPE_BUF)) 909 space = 0; 910 911 if (space > 0) { 912 int size; /* Transfer size */ 913 int segsize; /* first segment to transfer */ 914 915 /* 916 * Transfer size is minimum of uio transfer 917 * and free space in pipe buffer. 918 */ 919 if (space > uio->uio_resid) 920 size = uio->uio_resid; 921 else 922 size = space; 923 /* 924 * First segment to transfer is minimum of 925 * transfer size and contiguous space in 926 * pipe buffer. If first segment to transfer 927 * is less than the transfer size, we've got 928 * a wraparound in the buffer. 929 */ 930 segsize = bp->size - bp->in; 931 if (segsize > size) 932 segsize = size; 933 934 /* Transfer first segment */ 935 error = uiomove(&bp->buffer[bp->in], segsize, uio); 936 937 if (error == 0 && segsize < size) { 938 /* 939 * Transfer remaining part now, to 940 * support atomic writes. Wraparound 941 * happened. 942 */ 943 #ifdef DEBUG 944 if (bp->in + segsize != bp->size) 945 panic("Expected pipe buffer wraparound disappeared"); 946 #endif 947 948 error = uiomove(&bp->buffer[0], 949 size - segsize, uio); 950 } 951 if (error) 952 break; 953 954 bp->in += size; 955 if (bp->in >= bp->size) { 956 #ifdef DEBUG 957 if (bp->in != size - segsize + bp->size) 958 panic("Expected wraparound bad"); 959 #endif 960 bp->in = size - segsize; 961 } 962 963 bp->cnt += size; 964 #ifdef DEBUG 965 if (bp->cnt > bp->size) 966 panic("Pipe buffer overflow"); 967 #endif 968 } else { 969 /* 970 * If the "read-side" has been blocked, wake it up now. 971 */ 972 PIPE_LOCK(wpipe); 973 if (wpipe->pipe_state & PIPE_WANTR) { 974 wpipe->pipe_state &= ~PIPE_WANTR; 975 wakeup(wpipe); 976 } 977 PIPE_UNLOCK(wpipe); 978 979 /* 980 * don't block on non-blocking I/O 981 */ 982 if (fp->f_flag & FNONBLOCK) { 983 error = EAGAIN; 984 break; 985 } 986 987 /* 988 * We have no more space and have something to offer, 989 * wake up select/poll. 990 */ 991 if (bp->cnt) 992 pipeselwakeup(wpipe, wpipe, POLL_OUT); 993 994 PIPE_LOCK(wpipe); 995 pipeunlock(wpipe); 996 wpipe->pipe_state |= PIPE_WANTW; 997 error = ltsleep(wpipe, PSOCK | PCATCH, "pipewr", 0, 998 &wpipe->pipe_slock); 999 (void)pipelock(wpipe, 0); 1000 if (error != 0) 1001 break; 1002 /* 1003 * If read side wants to go away, we just issue a signal 1004 * to ourselves. 1005 */ 1006 if (wpipe->pipe_state & PIPE_EOF) { 1007 error = EPIPE; 1008 break; 1009 } 1010 } 1011 } 1012 1013 PIPE_LOCK(wpipe); 1014 --wpipe->pipe_busy; 1015 if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANTCLOSE)) { 1016 wpipe->pipe_state &= ~(PIPE_WANTCLOSE | PIPE_WANTR); 1017 wakeup(wpipe); 1018 } else if (bp->cnt > 0) { 1019 /* 1020 * If we have put any characters in the buffer, we wake up 1021 * the reader. 1022 */ 1023 if (wpipe->pipe_state & PIPE_WANTR) { 1024 wpipe->pipe_state &= ~PIPE_WANTR; 1025 wakeup(wpipe); 1026 } 1027 } 1028 1029 /* 1030 * Don't return EPIPE if I/O was successful 1031 */ 1032 if (error == EPIPE && bp->cnt == 0 && uio->uio_resid == 0) 1033 error = 0; 1034 1035 if (error == 0) 1036 getmicrotime(&wpipe->pipe_mtime); 1037 1038 /* 1039 * We have something to offer, wake up select/poll. 1040 * wpipe->pipe_map.cnt is always 0 in this point (direct write 1041 * is only done synchronously), so check only wpipe->pipe_buffer.cnt 1042 */ 1043 if (bp->cnt) 1044 pipeselwakeup(wpipe, wpipe, POLL_OUT); 1045 1046 /* 1047 * Arrange for next read(2) to do a signal. 1048 */ 1049 wpipe->pipe_state |= PIPE_SIGNALR; 1050 1051 pipeunlock(wpipe); 1052 PIPE_UNLOCK(wpipe); 1053 return (error); 1054 } 1055 1056 /* 1057 * we implement a very minimal set of ioctls for compatibility with sockets. 1058 */ 1059 int 1060 pipe_ioctl(struct file *fp, u_long cmd, void *data, struct lwp *l) 1061 { 1062 struct pipe *pipe = (struct pipe *)fp->f_data; 1063 struct proc *p = l->l_proc; 1064 1065 switch (cmd) { 1066 1067 case FIONBIO: 1068 return (0); 1069 1070 case FIOASYNC: 1071 PIPE_LOCK(pipe); 1072 if (*(int *)data) { 1073 pipe->pipe_state |= PIPE_ASYNC; 1074 } else { 1075 pipe->pipe_state &= ~PIPE_ASYNC; 1076 } 1077 PIPE_UNLOCK(pipe); 1078 return (0); 1079 1080 case FIONREAD: 1081 PIPE_LOCK(pipe); 1082 #ifndef PIPE_NODIRECT 1083 if (pipe->pipe_state & PIPE_DIRECTW) 1084 *(int *)data = pipe->pipe_map.cnt; 1085 else 1086 #endif 1087 *(int *)data = pipe->pipe_buffer.cnt; 1088 PIPE_UNLOCK(pipe); 1089 return (0); 1090 1091 case FIONWRITE: 1092 /* Look at other side */ 1093 pipe = pipe->pipe_peer; 1094 PIPE_LOCK(pipe); 1095 #ifndef PIPE_NODIRECT 1096 if (pipe->pipe_state & PIPE_DIRECTW) 1097 *(int *)data = pipe->pipe_map.cnt; 1098 else 1099 #endif 1100 *(int *)data = pipe->pipe_buffer.cnt; 1101 PIPE_UNLOCK(pipe); 1102 return (0); 1103 1104 case FIONSPACE: 1105 /* Look at other side */ 1106 pipe = pipe->pipe_peer; 1107 PIPE_LOCK(pipe); 1108 #ifndef PIPE_NODIRECT 1109 /* 1110 * If we're in direct-mode, we don't really have a 1111 * send queue, and any other write will block. Thus 1112 * zero seems like the best answer. 1113 */ 1114 if (pipe->pipe_state & PIPE_DIRECTW) 1115 *(int *)data = 0; 1116 else 1117 #endif 1118 *(int *)data = pipe->pipe_buffer.size - 1119 pipe->pipe_buffer.cnt; 1120 PIPE_UNLOCK(pipe); 1121 return (0); 1122 1123 case TIOCSPGRP: 1124 case FIOSETOWN: 1125 return fsetown(p, &pipe->pipe_pgid, cmd, data); 1126 1127 case TIOCGPGRP: 1128 case FIOGETOWN: 1129 return fgetown(p, pipe->pipe_pgid, cmd, data); 1130 1131 } 1132 return (EPASSTHROUGH); 1133 } 1134 1135 int 1136 pipe_poll(struct file *fp, int events, struct lwp *l) 1137 { 1138 struct pipe *rpipe = (struct pipe *)fp->f_data; 1139 struct pipe *wpipe; 1140 int eof = 0; 1141 int revents = 0; 1142 1143 retry: 1144 PIPE_LOCK(rpipe); 1145 wpipe = rpipe->pipe_peer; 1146 if (wpipe != NULL && simple_lock_try(&wpipe->pipe_slock) == 0) { 1147 /* Deal with race for peer */ 1148 PIPE_UNLOCK(rpipe); 1149 goto retry; 1150 } 1151 1152 if (events & (POLLIN | POLLRDNORM)) 1153 if ((rpipe->pipe_buffer.cnt > 0) || 1154 #ifndef PIPE_NODIRECT 1155 (rpipe->pipe_state & PIPE_DIRECTR) || 1156 #endif 1157 (rpipe->pipe_state & PIPE_EOF)) 1158 revents |= events & (POLLIN | POLLRDNORM); 1159 1160 eof |= (rpipe->pipe_state & PIPE_EOF); 1161 PIPE_UNLOCK(rpipe); 1162 1163 if (wpipe == NULL) 1164 revents |= events & (POLLOUT | POLLWRNORM); 1165 else { 1166 if (events & (POLLOUT | POLLWRNORM)) 1167 if ((wpipe->pipe_state & PIPE_EOF) || ( 1168 #ifndef PIPE_NODIRECT 1169 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 1170 #endif 1171 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 1172 revents |= events & (POLLOUT | POLLWRNORM); 1173 1174 eof |= (wpipe->pipe_state & PIPE_EOF); 1175 PIPE_UNLOCK(wpipe); 1176 } 1177 1178 if (wpipe == NULL || eof) 1179 revents |= POLLHUP; 1180 1181 if (revents == 0) { 1182 if (events & (POLLIN | POLLRDNORM)) 1183 selrecord(l, &rpipe->pipe_sel); 1184 1185 if (events & (POLLOUT | POLLWRNORM)) 1186 selrecord(l, &wpipe->pipe_sel); 1187 } 1188 1189 return (revents); 1190 } 1191 1192 static int 1193 pipe_stat(struct file *fp, struct stat *ub, struct lwp *l __unused) 1194 { 1195 struct pipe *pipe = (struct pipe *)fp->f_data; 1196 1197 memset((caddr_t)ub, 0, sizeof(*ub)); 1198 ub->st_mode = S_IFIFO | S_IRUSR | S_IWUSR; 1199 ub->st_blksize = pipe->pipe_buffer.size; 1200 if (ub->st_blksize == 0 && pipe->pipe_peer) 1201 ub->st_blksize = pipe->pipe_peer->pipe_buffer.size; 1202 ub->st_size = pipe->pipe_buffer.cnt; 1203 ub->st_blocks = (ub->st_size) ? 1 : 0; 1204 TIMEVAL_TO_TIMESPEC(&pipe->pipe_atime, &ub->st_atimespec); 1205 TIMEVAL_TO_TIMESPEC(&pipe->pipe_mtime, &ub->st_mtimespec); 1206 TIMEVAL_TO_TIMESPEC(&pipe->pipe_ctime, &ub->st_ctimespec); 1207 ub->st_uid = kauth_cred_geteuid(fp->f_cred); 1208 ub->st_gid = kauth_cred_getegid(fp->f_cred); 1209 /* 1210 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 1211 * XXX (st_dev, st_ino) should be unique. 1212 */ 1213 return (0); 1214 } 1215 1216 /* ARGSUSED */ 1217 static int 1218 pipe_close(struct file *fp, struct lwp *l __unused) 1219 { 1220 struct pipe *pipe = (struct pipe *)fp->f_data; 1221 1222 fp->f_data = NULL; 1223 pipeclose(fp, pipe); 1224 return (0); 1225 } 1226 1227 static void 1228 pipe_free_kmem(struct pipe *pipe) 1229 { 1230 1231 if (pipe->pipe_buffer.buffer != NULL) { 1232 if (pipe->pipe_buffer.size > PIPE_SIZE) 1233 --nbigpipe; 1234 amountpipekva -= pipe->pipe_buffer.size; 1235 uvm_km_free(kernel_map, 1236 (vaddr_t)pipe->pipe_buffer.buffer, 1237 pipe->pipe_buffer.size, UVM_KMF_PAGEABLE); 1238 pipe->pipe_buffer.buffer = NULL; 1239 } 1240 #ifndef PIPE_NODIRECT 1241 if (pipe->pipe_map.kva != 0) { 1242 pipe_loan_free(pipe); 1243 pipe->pipe_map.cnt = 0; 1244 pipe->pipe_map.kva = 0; 1245 pipe->pipe_map.pos = 0; 1246 pipe->pipe_map.npages = 0; 1247 } 1248 #endif /* !PIPE_NODIRECT */ 1249 } 1250 1251 /* 1252 * shutdown the pipe 1253 */ 1254 static void 1255 pipeclose(struct file *fp __unused, struct pipe *pipe) 1256 { 1257 struct pipe *ppipe; 1258 1259 if (pipe == NULL) 1260 return; 1261 1262 retry: 1263 PIPE_LOCK(pipe); 1264 1265 pipeselwakeup(pipe, pipe, POLL_HUP); 1266 1267 /* 1268 * If the other side is blocked, wake it up saying that 1269 * we want to close it down. 1270 */ 1271 pipe->pipe_state |= PIPE_EOF; 1272 while (pipe->pipe_busy) { 1273 wakeup(pipe); 1274 pipe->pipe_state |= PIPE_WANTCLOSE; 1275 ltsleep(pipe, PSOCK, "pipecl", 0, &pipe->pipe_slock); 1276 } 1277 1278 /* 1279 * Disconnect from peer 1280 */ 1281 if ((ppipe = pipe->pipe_peer) != NULL) { 1282 /* Deal with race for peer */ 1283 if (simple_lock_try(&ppipe->pipe_slock) == 0) { 1284 PIPE_UNLOCK(pipe); 1285 goto retry; 1286 } 1287 pipeselwakeup(ppipe, ppipe, POLL_HUP); 1288 1289 ppipe->pipe_state |= PIPE_EOF; 1290 wakeup(ppipe); 1291 ppipe->pipe_peer = NULL; 1292 PIPE_UNLOCK(ppipe); 1293 } 1294 1295 KASSERT((pipe->pipe_state & PIPE_LOCKFL) == 0); 1296 1297 PIPE_UNLOCK(pipe); 1298 1299 /* 1300 * free resources 1301 */ 1302 pipe_free_kmem(pipe); 1303 pool_put(&pipe_pool, pipe); 1304 } 1305 1306 static void 1307 filt_pipedetach(struct knote *kn) 1308 { 1309 struct pipe *pipe = (struct pipe *)kn->kn_fp->f_data; 1310 1311 switch(kn->kn_filter) { 1312 case EVFILT_WRITE: 1313 /* need the peer structure, not our own */ 1314 pipe = pipe->pipe_peer; 1315 /* XXXSMP: race for peer */ 1316 1317 /* if reader end already closed, just return */ 1318 if (pipe == NULL) 1319 return; 1320 1321 break; 1322 default: 1323 /* nothing to do */ 1324 break; 1325 } 1326 1327 #ifdef DIAGNOSTIC 1328 if (kn->kn_hook != pipe) 1329 panic("filt_pipedetach: inconsistent knote"); 1330 #endif 1331 1332 PIPE_LOCK(pipe); 1333 SLIST_REMOVE(&pipe->pipe_sel.sel_klist, kn, knote, kn_selnext); 1334 PIPE_UNLOCK(pipe); 1335 } 1336 1337 /*ARGSUSED*/ 1338 static int 1339 filt_piperead(struct knote *kn, long hint) 1340 { 1341 struct pipe *rpipe = (struct pipe *)kn->kn_fp->f_data; 1342 struct pipe *wpipe = rpipe->pipe_peer; 1343 1344 if ((hint & NOTE_SUBMIT) == 0) 1345 PIPE_LOCK(rpipe); 1346 kn->kn_data = rpipe->pipe_buffer.cnt; 1347 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1348 kn->kn_data = rpipe->pipe_map.cnt; 1349 1350 /* XXXSMP: race for peer */ 1351 if ((rpipe->pipe_state & PIPE_EOF) || 1352 (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1353 kn->kn_flags |= EV_EOF; 1354 if ((hint & NOTE_SUBMIT) == 0) 1355 PIPE_UNLOCK(rpipe); 1356 return (1); 1357 } 1358 if ((hint & NOTE_SUBMIT) == 0) 1359 PIPE_UNLOCK(rpipe); 1360 return (kn->kn_data > 0); 1361 } 1362 1363 /*ARGSUSED*/ 1364 static int 1365 filt_pipewrite(struct knote *kn, long hint) 1366 { 1367 struct pipe *rpipe = (struct pipe *)kn->kn_fp->f_data; 1368 struct pipe *wpipe = rpipe->pipe_peer; 1369 1370 if ((hint & NOTE_SUBMIT) == 0) 1371 PIPE_LOCK(rpipe); 1372 /* XXXSMP: race for peer */ 1373 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1374 kn->kn_data = 0; 1375 kn->kn_flags |= EV_EOF; 1376 if ((hint & NOTE_SUBMIT) == 0) 1377 PIPE_UNLOCK(rpipe); 1378 return (1); 1379 } 1380 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1381 if (wpipe->pipe_state & PIPE_DIRECTW) 1382 kn->kn_data = 0; 1383 1384 if ((hint & NOTE_SUBMIT) == 0) 1385 PIPE_UNLOCK(rpipe); 1386 return (kn->kn_data >= PIPE_BUF); 1387 } 1388 1389 static const struct filterops pipe_rfiltops = 1390 { 1, NULL, filt_pipedetach, filt_piperead }; 1391 static const struct filterops pipe_wfiltops = 1392 { 1, NULL, filt_pipedetach, filt_pipewrite }; 1393 1394 /*ARGSUSED*/ 1395 static int 1396 pipe_kqfilter(struct file *fp __unused, struct knote *kn) 1397 { 1398 struct pipe *pipe; 1399 1400 pipe = (struct pipe *)kn->kn_fp->f_data; 1401 switch (kn->kn_filter) { 1402 case EVFILT_READ: 1403 kn->kn_fop = &pipe_rfiltops; 1404 break; 1405 case EVFILT_WRITE: 1406 kn->kn_fop = &pipe_wfiltops; 1407 /* XXXSMP: race for peer */ 1408 pipe = pipe->pipe_peer; 1409 if (pipe == NULL) { 1410 /* other end of pipe has been closed */ 1411 return (EBADF); 1412 } 1413 break; 1414 default: 1415 return (1); 1416 } 1417 kn->kn_hook = pipe; 1418 1419 PIPE_LOCK(pipe); 1420 SLIST_INSERT_HEAD(&pipe->pipe_sel.sel_klist, kn, kn_selnext); 1421 PIPE_UNLOCK(pipe); 1422 return (0); 1423 } 1424 1425 /* 1426 * Handle pipe sysctls. 1427 */ 1428 SYSCTL_SETUP(sysctl_kern_pipe_setup, "sysctl kern.pipe subtree setup") 1429 { 1430 1431 sysctl_createv(clog, 0, NULL, NULL, 1432 CTLFLAG_PERMANENT, 1433 CTLTYPE_NODE, "kern", NULL, 1434 NULL, 0, NULL, 0, 1435 CTL_KERN, CTL_EOL); 1436 sysctl_createv(clog, 0, NULL, NULL, 1437 CTLFLAG_PERMANENT, 1438 CTLTYPE_NODE, "pipe", 1439 SYSCTL_DESCR("Pipe settings"), 1440 NULL, 0, NULL, 0, 1441 CTL_KERN, KERN_PIPE, CTL_EOL); 1442 1443 sysctl_createv(clog, 0, NULL, NULL, 1444 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1445 CTLTYPE_INT, "maxkvasz", 1446 SYSCTL_DESCR("Maximum amount of kernel memory to be " 1447 "used for pipes"), 1448 NULL, 0, &maxpipekva, 0, 1449 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXKVASZ, CTL_EOL); 1450 sysctl_createv(clog, 0, NULL, NULL, 1451 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1452 CTLTYPE_INT, "maxloankvasz", 1453 SYSCTL_DESCR("Limit for direct transfers via page loan"), 1454 NULL, 0, &limitpipekva, 0, 1455 CTL_KERN, KERN_PIPE, KERN_PIPE_LIMITKVA, CTL_EOL); 1456 sysctl_createv(clog, 0, NULL, NULL, 1457 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1458 CTLTYPE_INT, "maxbigpipes", 1459 SYSCTL_DESCR("Maximum number of \"big\" pipes"), 1460 NULL, 0, &maxbigpipes, 0, 1461 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXBIGPIPES, CTL_EOL); 1462 sysctl_createv(clog, 0, NULL, NULL, 1463 CTLFLAG_PERMANENT, 1464 CTLTYPE_INT, "nbigpipes", 1465 SYSCTL_DESCR("Number of \"big\" pipes"), 1466 NULL, 0, &nbigpipe, 0, 1467 CTL_KERN, KERN_PIPE, KERN_PIPE_NBIGPIPES, CTL_EOL); 1468 sysctl_createv(clog, 0, NULL, NULL, 1469 CTLFLAG_PERMANENT, 1470 CTLTYPE_INT, "kvasize", 1471 SYSCTL_DESCR("Amount of kernel memory consumed by pipe " 1472 "buffers"), 1473 NULL, 0, &amountpipekva, 0, 1474 CTL_KERN, KERN_PIPE, KERN_PIPE_KVASIZE, CTL_EOL); 1475 } 1476