1 /* $NetBSD: sys_pipe.c,v 1.63 2005/02/26 21:34:55 perry Exp $ */ 2 3 /*- 4 * Copyright (c) 2003 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Paul Kranenburg. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Copyright (c) 1996 John S. Dyson 41 * All rights reserved. 42 * 43 * Redistribution and use in source and binary forms, with or without 44 * modification, are permitted provided that the following conditions 45 * are met: 46 * 1. Redistributions of source code must retain the above copyright 47 * notice immediately at the beginning of the file, without modification, 48 * this list of conditions, and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. Absolutely no warranty of function or purpose is made by the author 53 * John S. Dyson. 54 * 4. Modifications may be freely made to this file if the above conditions 55 * are met. 56 * 57 * $FreeBSD: src/sys/kern/sys_pipe.c,v 1.95 2002/03/09 22:06:31 alfred Exp $ 58 */ 59 60 /* 61 * This file contains a high-performance replacement for the socket-based 62 * pipes scheme originally used in FreeBSD/4.4Lite. It does not support 63 * all features of sockets, but does do everything that pipes normally 64 * do. 65 * 66 * Adaption for NetBSD UVM, including uvm_loan() based direct write, was 67 * written by Jaromir Dolecek. 68 */ 69 70 /* 71 * This code has two modes of operation, a small write mode and a large 72 * write mode. The small write mode acts like conventional pipes with 73 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 74 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 75 * and PIPE_SIZE in size it is mapped read-only into the kernel address space 76 * using the UVM page loan facility from where the receiving process can copy 77 * the data directly from the pages in the sending process. 78 * 79 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 80 * happen for small transfers so that the system will not spend all of 81 * its time context switching. PIPE_SIZE is constrained by the 82 * amount of kernel virtual memory. 83 */ 84 85 #include <sys/cdefs.h> 86 __KERNEL_RCSID(0, "$NetBSD: sys_pipe.c,v 1.63 2005/02/26 21:34:55 perry Exp $"); 87 88 #include <sys/param.h> 89 #include <sys/systm.h> 90 #include <sys/proc.h> 91 #include <sys/fcntl.h> 92 #include <sys/file.h> 93 #include <sys/filedesc.h> 94 #include <sys/filio.h> 95 #include <sys/kernel.h> 96 #include <sys/lock.h> 97 #include <sys/ttycom.h> 98 #include <sys/stat.h> 99 #include <sys/malloc.h> 100 #include <sys/poll.h> 101 #include <sys/signalvar.h> 102 #include <sys/vnode.h> 103 #include <sys/uio.h> 104 #include <sys/lock.h> 105 #include <sys/select.h> 106 #include <sys/mount.h> 107 #include <sys/sa.h> 108 #include <sys/syscallargs.h> 109 #include <uvm/uvm.h> 110 #include <sys/sysctl.h> 111 #include <sys/kernel.h> 112 113 #include <sys/pipe.h> 114 115 /* 116 * Avoid microtime(9), it's slow. We don't guard the read from time(9) 117 * with splclock(9) since we don't actually need to be THAT sure the access 118 * is atomic. 119 */ 120 #define PIPE_TIMESTAMP(tvp) (*(tvp) = time) 121 122 123 /* 124 * Use this define if you want to disable *fancy* VM things. Expect an 125 * approx 30% decrease in transfer rate. 126 */ 127 /* #define PIPE_NODIRECT */ 128 129 /* 130 * interfaces to the outside world 131 */ 132 static int pipe_read(struct file *fp, off_t *offset, struct uio *uio, 133 struct ucred *cred, int flags); 134 static int pipe_write(struct file *fp, off_t *offset, struct uio *uio, 135 struct ucred *cred, int flags); 136 static int pipe_close(struct file *fp, struct proc *p); 137 static int pipe_poll(struct file *fp, int events, struct proc *p); 138 static int pipe_kqfilter(struct file *fp, struct knote *kn); 139 static int pipe_stat(struct file *fp, struct stat *sb, struct proc *p); 140 static int pipe_ioctl(struct file *fp, u_long cmd, void *data, 141 struct proc *p); 142 143 static const struct fileops pipeops = { 144 pipe_read, pipe_write, pipe_ioctl, fnullop_fcntl, pipe_poll, 145 pipe_stat, pipe_close, pipe_kqfilter 146 }; 147 148 /* 149 * Default pipe buffer size(s), this can be kind-of large now because pipe 150 * space is pageable. The pipe code will try to maintain locality of 151 * reference for performance reasons, so small amounts of outstanding I/O 152 * will not wipe the cache. 153 */ 154 #define MINPIPESIZE (PIPE_SIZE/3) 155 #define MAXPIPESIZE (2*PIPE_SIZE/3) 156 157 /* 158 * Maximum amount of kva for pipes -- this is kind-of a soft limit, but 159 * is there so that on large systems, we don't exhaust it. 160 */ 161 #define MAXPIPEKVA (8*1024*1024) 162 static int maxpipekva = MAXPIPEKVA; 163 164 /* 165 * Limit for direct transfers, we cannot, of course limit 166 * the amount of kva for pipes in general though. 167 */ 168 #define LIMITPIPEKVA (16*1024*1024) 169 static int limitpipekva = LIMITPIPEKVA; 170 171 /* 172 * Limit the number of "big" pipes 173 */ 174 #define LIMITBIGPIPES 32 175 static int maxbigpipes = LIMITBIGPIPES; 176 static int nbigpipe = 0; 177 178 /* 179 * Amount of KVA consumed by pipe buffers. 180 */ 181 static int amountpipekva = 0; 182 183 MALLOC_DEFINE(M_PIPE, "pipe", "Pipe structures"); 184 185 static void pipeclose(struct file *fp, struct pipe *pipe); 186 static void pipe_free_kmem(struct pipe *pipe); 187 static int pipe_create(struct pipe **pipep, int allockva); 188 static int pipelock(struct pipe *pipe, int catch); 189 static __inline void pipeunlock(struct pipe *pipe); 190 static void pipeselwakeup(struct pipe *pipe, struct pipe *sigp, void *data, 191 int code); 192 #ifndef PIPE_NODIRECT 193 static int pipe_direct_write(struct file *fp, struct pipe *wpipe, 194 struct uio *uio); 195 #endif 196 static int pipespace(struct pipe *pipe, int size); 197 198 #ifndef PIPE_NODIRECT 199 static int pipe_loan_alloc(struct pipe *, int); 200 static void pipe_loan_free(struct pipe *); 201 #endif /* PIPE_NODIRECT */ 202 203 static POOL_INIT(pipe_pool, sizeof(struct pipe), 0, 0, 0, "pipepl", 204 &pool_allocator_nointr); 205 206 /* 207 * The pipe system call for the DTYPE_PIPE type of pipes 208 */ 209 210 /* ARGSUSED */ 211 int 212 sys_pipe(l, v, retval) 213 struct lwp *l; 214 void *v; 215 register_t *retval; 216 { 217 struct file *rf, *wf; 218 struct pipe *rpipe, *wpipe; 219 int fd, error; 220 struct proc *p; 221 222 p = l->l_proc; 223 rpipe = wpipe = NULL; 224 if (pipe_create(&rpipe, 1) || pipe_create(&wpipe, 0)) { 225 pipeclose(NULL, rpipe); 226 pipeclose(NULL, wpipe); 227 return (ENFILE); 228 } 229 230 /* 231 * Note: the file structure returned from falloc() is marked 232 * as 'larval' initially. Unless we mark it as 'mature' by 233 * FILE_SET_MATURE(), any attempt to do anything with it would 234 * return EBADF, including e.g. dup(2) or close(2). This avoids 235 * file descriptor races if we block in the second falloc(). 236 */ 237 238 error = falloc(p, &rf, &fd); 239 if (error) 240 goto free2; 241 retval[0] = fd; 242 rf->f_flag = FREAD; 243 rf->f_type = DTYPE_PIPE; 244 rf->f_data = (caddr_t)rpipe; 245 rf->f_ops = &pipeops; 246 247 error = falloc(p, &wf, &fd); 248 if (error) 249 goto free3; 250 retval[1] = fd; 251 wf->f_flag = FWRITE; 252 wf->f_type = DTYPE_PIPE; 253 wf->f_data = (caddr_t)wpipe; 254 wf->f_ops = &pipeops; 255 256 rpipe->pipe_peer = wpipe; 257 wpipe->pipe_peer = rpipe; 258 259 FILE_SET_MATURE(rf); 260 FILE_SET_MATURE(wf); 261 FILE_UNUSE(rf, p); 262 FILE_UNUSE(wf, p); 263 return (0); 264 free3: 265 FILE_UNUSE(rf, p); 266 ffree(rf); 267 fdremove(p->p_fd, retval[0]); 268 free2: 269 pipeclose(NULL, wpipe); 270 pipeclose(NULL, rpipe); 271 272 return (error); 273 } 274 275 /* 276 * Allocate kva for pipe circular buffer, the space is pageable 277 * This routine will 'realloc' the size of a pipe safely, if it fails 278 * it will retain the old buffer. 279 * If it fails it will return ENOMEM. 280 */ 281 static int 282 pipespace(pipe, size) 283 struct pipe *pipe; 284 int size; 285 { 286 caddr_t buffer; 287 /* 288 * Allocate pageable virtual address space. Physical memory is 289 * allocated on demand. 290 */ 291 buffer = (caddr_t) uvm_km_valloc(kernel_map, round_page(size)); 292 if (buffer == NULL) 293 return (ENOMEM); 294 295 /* free old resources if we're resizing */ 296 pipe_free_kmem(pipe); 297 pipe->pipe_buffer.buffer = buffer; 298 pipe->pipe_buffer.size = size; 299 pipe->pipe_buffer.in = 0; 300 pipe->pipe_buffer.out = 0; 301 pipe->pipe_buffer.cnt = 0; 302 amountpipekva += pipe->pipe_buffer.size; 303 return (0); 304 } 305 306 /* 307 * Initialize and allocate VM and memory for pipe. 308 */ 309 static int 310 pipe_create(pipep, allockva) 311 struct pipe **pipep; 312 int allockva; 313 { 314 struct pipe *pipe; 315 int error; 316 317 pipe = *pipep = pool_get(&pipe_pool, PR_WAITOK); 318 319 /* Initialize */ 320 memset(pipe, 0, sizeof(struct pipe)); 321 pipe->pipe_state = PIPE_SIGNALR; 322 323 PIPE_TIMESTAMP(&pipe->pipe_ctime); 324 pipe->pipe_atime = pipe->pipe_ctime; 325 pipe->pipe_mtime = pipe->pipe_ctime; 326 simple_lock_init(&pipe->pipe_slock); 327 lockinit(&pipe->pipe_lock, PSOCK | PCATCH, "pipelk", 0, 0); 328 329 if (allockva && (error = pipespace(pipe, PIPE_SIZE))) 330 return (error); 331 332 return (0); 333 } 334 335 336 /* 337 * Lock a pipe for I/O, blocking other access 338 * Called with pipe spin lock held. 339 * Return with pipe spin lock released on success. 340 */ 341 static int 342 pipelock(pipe, catch) 343 struct pipe *pipe; 344 int catch; 345 { 346 int error; 347 348 LOCK_ASSERT(simple_lock_held(&pipe->pipe_slock)); 349 350 while (1) { 351 error = lockmgr(&pipe->pipe_lock, LK_EXCLUSIVE | LK_INTERLOCK, 352 &pipe->pipe_slock); 353 if (error == 0) 354 break; 355 356 simple_lock(&pipe->pipe_slock); 357 if (catch || (error != EINTR && error != ERESTART)) 358 break; 359 /* 360 * XXX XXX XXX 361 * The pipe lock is initialised with PCATCH on and we cannot 362 * override this in a lockmgr() call. Thus a pending signal 363 * will cause lockmgr() to return with EINTR or ERESTART. 364 * We cannot simply re-enter lockmgr() at this point since 365 * the pending signals have not yet been posted and would 366 * cause an immediate EINTR/ERESTART return again. 367 * As a workaround we pause for a while here, giving the lock 368 * a chance to drain, before trying again. 369 * XXX XXX XXX 370 * 371 * NOTE: Consider dropping PCATCH from this lock; in practice 372 * it is never held for long enough periods for having it 373 * interruptable at the start of pipe_read/pipe_write to be 374 * beneficial. 375 */ 376 (void) ltsleep(&lbolt, PSOCK, "rstrtpipelock", hz, 377 &pipe->pipe_slock); 378 } 379 return (error); 380 } 381 382 /* 383 * unlock a pipe I/O lock 384 */ 385 static __inline void 386 pipeunlock(pipe) 387 struct pipe *pipe; 388 { 389 390 lockmgr(&pipe->pipe_lock, LK_RELEASE, NULL); 391 } 392 393 /* 394 * Select/poll wakup. This also sends SIGIO to peer connected to 395 * 'sigpipe' side of pipe. 396 */ 397 static void 398 pipeselwakeup(selp, sigp, data, code) 399 struct pipe *selp, *sigp; 400 void *data; 401 int code; 402 { 403 int band; 404 405 selnotify(&selp->pipe_sel, NOTE_SUBMIT); 406 407 if (sigp == NULL || (sigp->pipe_state & PIPE_ASYNC) == 0) 408 return; 409 410 switch (code) { 411 case POLL_IN: 412 band = POLLIN|POLLRDNORM; 413 break; 414 case POLL_OUT: 415 band = POLLOUT|POLLWRNORM; 416 break; 417 case POLL_HUP: 418 band = POLLHUP; 419 break; 420 #if POLL_HUP != POLL_ERR 421 case POLL_ERR: 422 band = POLLERR; 423 break; 424 #endif 425 default: 426 band = 0; 427 #ifdef DIAGNOSTIC 428 printf("bad siginfo code %d in pipe notification.\n", code); 429 #endif 430 break; 431 } 432 433 fownsignal(sigp->pipe_pgid, SIGIO, code, band, selp); 434 } 435 436 /* ARGSUSED */ 437 static int 438 pipe_read(fp, offset, uio, cred, flags) 439 struct file *fp; 440 off_t *offset; 441 struct uio *uio; 442 struct ucred *cred; 443 int flags; 444 { 445 struct pipe *rpipe = (struct pipe *) fp->f_data; 446 struct pipebuf *bp = &rpipe->pipe_buffer; 447 int error; 448 size_t nread = 0; 449 size_t size; 450 size_t ocnt; 451 452 PIPE_LOCK(rpipe); 453 ++rpipe->pipe_busy; 454 ocnt = bp->cnt; 455 456 again: 457 error = pipelock(rpipe, 1); 458 if (error) 459 goto unlocked_error; 460 461 while (uio->uio_resid) { 462 /* 463 * normal pipe buffer receive 464 */ 465 if (bp->cnt > 0) { 466 size = bp->size - bp->out; 467 if (size > bp->cnt) 468 size = bp->cnt; 469 if (size > uio->uio_resid) 470 size = uio->uio_resid; 471 472 error = uiomove(&bp->buffer[bp->out], size, uio); 473 if (error) 474 break; 475 476 bp->out += size; 477 if (bp->out >= bp->size) 478 bp->out = 0; 479 480 bp->cnt -= size; 481 482 /* 483 * If there is no more to read in the pipe, reset 484 * its pointers to the beginning. This improves 485 * cache hit stats. 486 */ 487 if (bp->cnt == 0) { 488 bp->in = 0; 489 bp->out = 0; 490 } 491 nread += size; 492 #ifndef PIPE_NODIRECT 493 } else if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) { 494 /* 495 * Direct copy, bypassing a kernel buffer. 496 */ 497 caddr_t va; 498 499 KASSERT(rpipe->pipe_state & PIPE_DIRECTW); 500 501 size = rpipe->pipe_map.cnt; 502 if (size > uio->uio_resid) 503 size = uio->uio_resid; 504 505 va = (caddr_t) rpipe->pipe_map.kva + 506 rpipe->pipe_map.pos; 507 error = uiomove(va, size, uio); 508 if (error) 509 break; 510 nread += size; 511 rpipe->pipe_map.pos += size; 512 rpipe->pipe_map.cnt -= size; 513 if (rpipe->pipe_map.cnt == 0) { 514 PIPE_LOCK(rpipe); 515 rpipe->pipe_state &= ~PIPE_DIRECTR; 516 wakeup(rpipe); 517 PIPE_UNLOCK(rpipe); 518 } 519 #endif 520 } else { 521 /* 522 * Break if some data was read. 523 */ 524 if (nread > 0) 525 break; 526 527 PIPE_LOCK(rpipe); 528 529 /* 530 * detect EOF condition 531 * read returns 0 on EOF, no need to set error 532 */ 533 if (rpipe->pipe_state & PIPE_EOF) { 534 PIPE_UNLOCK(rpipe); 535 break; 536 } 537 538 /* 539 * don't block on non-blocking I/O 540 */ 541 if (fp->f_flag & FNONBLOCK) { 542 PIPE_UNLOCK(rpipe); 543 error = EAGAIN; 544 break; 545 } 546 547 /* 548 * Unlock the pipe buffer for our remaining processing. 549 * We will either break out with an error or we will 550 * sleep and relock to loop. 551 */ 552 pipeunlock(rpipe); 553 554 /* 555 * The PIPE_DIRECTR flag is not under the control 556 * of the long-term lock (see pipe_direct_write()), 557 * so re-check now while holding the spin lock. 558 */ 559 if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) 560 goto again; 561 562 /* 563 * We want to read more, wake up select/poll. 564 */ 565 pipeselwakeup(rpipe, rpipe->pipe_peer, fp->f_data, 566 POLL_IN); 567 568 /* 569 * If the "write-side" is blocked, wake it up now. 570 */ 571 if (rpipe->pipe_state & PIPE_WANTW) { 572 rpipe->pipe_state &= ~PIPE_WANTW; 573 wakeup(rpipe); 574 } 575 576 /* Now wait until the pipe is filled */ 577 rpipe->pipe_state |= PIPE_WANTR; 578 error = ltsleep(rpipe, PSOCK | PCATCH, 579 "piperd", 0, &rpipe->pipe_slock); 580 if (error != 0) 581 goto unlocked_error; 582 goto again; 583 } 584 } 585 586 if (error == 0) 587 PIPE_TIMESTAMP(&rpipe->pipe_atime); 588 589 PIPE_LOCK(rpipe); 590 pipeunlock(rpipe); 591 592 unlocked_error: 593 --rpipe->pipe_busy; 594 595 /* 596 * PIPE_WANTCLOSE processing only makes sense if pipe_busy is 0. 597 */ 598 if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANTCLOSE)) { 599 rpipe->pipe_state &= ~(PIPE_WANTCLOSE|PIPE_WANTW); 600 wakeup(rpipe); 601 } else if (bp->cnt < MINPIPESIZE) { 602 /* 603 * Handle write blocking hysteresis. 604 */ 605 if (rpipe->pipe_state & PIPE_WANTW) { 606 rpipe->pipe_state &= ~PIPE_WANTW; 607 wakeup(rpipe); 608 } 609 } 610 611 /* 612 * If anything was read off the buffer, signal to the writer it's 613 * possible to write more data. Also send signal if we are here for the 614 * first time after last write. 615 */ 616 if ((bp->size - bp->cnt) >= PIPE_BUF 617 && (ocnt != bp->cnt || (rpipe->pipe_state & PIPE_SIGNALR))) { 618 pipeselwakeup(rpipe, rpipe->pipe_peer, fp->f_data, POLL_OUT); 619 rpipe->pipe_state &= ~PIPE_SIGNALR; 620 } 621 622 PIPE_UNLOCK(rpipe); 623 return (error); 624 } 625 626 #ifndef PIPE_NODIRECT 627 /* 628 * Allocate structure for loan transfer. 629 */ 630 static int 631 pipe_loan_alloc(wpipe, npages) 632 struct pipe *wpipe; 633 int npages; 634 { 635 vsize_t len; 636 637 len = (vsize_t)npages << PAGE_SHIFT; 638 wpipe->pipe_map.kva = uvm_km_valloc_wait(kernel_map, len); 639 if (wpipe->pipe_map.kva == 0) 640 return (ENOMEM); 641 642 amountpipekva += len; 643 wpipe->pipe_map.npages = npages; 644 wpipe->pipe_map.pgs = malloc(npages * sizeof(struct vm_page *), M_PIPE, 645 M_WAITOK); 646 return (0); 647 } 648 649 /* 650 * Free resources allocated for loan transfer. 651 */ 652 static void 653 pipe_loan_free(wpipe) 654 struct pipe *wpipe; 655 { 656 vsize_t len; 657 658 len = (vsize_t)wpipe->pipe_map.npages << PAGE_SHIFT; 659 uvm_km_free(kernel_map, wpipe->pipe_map.kva, len); 660 wpipe->pipe_map.kva = 0; 661 amountpipekva -= len; 662 free(wpipe->pipe_map.pgs, M_PIPE); 663 wpipe->pipe_map.pgs = NULL; 664 } 665 666 /* 667 * NetBSD direct write, using uvm_loan() mechanism. 668 * This implements the pipe buffer write mechanism. Note that only 669 * a direct write OR a normal pipe write can be pending at any given time. 670 * If there are any characters in the pipe buffer, the direct write will 671 * be deferred until the receiving process grabs all of the bytes from 672 * the pipe buffer. Then the direct mapping write is set-up. 673 * 674 * Called with the long-term pipe lock held. 675 */ 676 static int 677 pipe_direct_write(fp, wpipe, uio) 678 struct file *fp; 679 struct pipe *wpipe; 680 struct uio *uio; 681 { 682 int error, npages, j; 683 struct vm_page **pgs; 684 vaddr_t bbase, kva, base, bend; 685 vsize_t blen, bcnt; 686 voff_t bpos; 687 688 KASSERT(wpipe->pipe_map.cnt == 0); 689 690 /* 691 * Handle first PIPE_CHUNK_SIZE bytes of buffer. Deal with buffers 692 * not aligned to PAGE_SIZE. 693 */ 694 bbase = (vaddr_t)uio->uio_iov->iov_base; 695 base = trunc_page(bbase); 696 bend = round_page(bbase + uio->uio_iov->iov_len); 697 blen = bend - base; 698 bpos = bbase - base; 699 700 if (blen > PIPE_DIRECT_CHUNK) { 701 blen = PIPE_DIRECT_CHUNK; 702 bend = base + blen; 703 bcnt = PIPE_DIRECT_CHUNK - bpos; 704 } else { 705 bcnt = uio->uio_iov->iov_len; 706 } 707 npages = blen >> PAGE_SHIFT; 708 709 /* 710 * Free the old kva if we need more pages than we have 711 * allocated. 712 */ 713 if (wpipe->pipe_map.kva != 0 && npages > wpipe->pipe_map.npages) 714 pipe_loan_free(wpipe); 715 716 /* Allocate new kva. */ 717 if (wpipe->pipe_map.kva == 0) { 718 error = pipe_loan_alloc(wpipe, npages); 719 if (error) 720 return (error); 721 } 722 723 /* Loan the write buffer memory from writer process */ 724 pgs = wpipe->pipe_map.pgs; 725 error = uvm_loan(&uio->uio_procp->p_vmspace->vm_map, base, blen, 726 pgs, UVM_LOAN_TOPAGE); 727 if (error) { 728 pipe_loan_free(wpipe); 729 return (ENOMEM); /* so that caller fallback to ordinary write */ 730 } 731 732 /* Enter the loaned pages to kva */ 733 kva = wpipe->pipe_map.kva; 734 for (j = 0; j < npages; j++, kva += PAGE_SIZE) { 735 pmap_kenter_pa(kva, VM_PAGE_TO_PHYS(pgs[j]), VM_PROT_READ); 736 } 737 pmap_update(pmap_kernel()); 738 739 /* Now we can put the pipe in direct write mode */ 740 wpipe->pipe_map.pos = bpos; 741 wpipe->pipe_map.cnt = bcnt; 742 wpipe->pipe_state |= PIPE_DIRECTW; 743 744 /* 745 * But before we can let someone do a direct read, 746 * we have to wait until the pipe is drained. 747 */ 748 749 /* Relase the pipe lock while we wait */ 750 PIPE_LOCK(wpipe); 751 pipeunlock(wpipe); 752 753 while (error == 0 && wpipe->pipe_buffer.cnt > 0) { 754 if (wpipe->pipe_state & PIPE_WANTR) { 755 wpipe->pipe_state &= ~PIPE_WANTR; 756 wakeup(wpipe); 757 } 758 759 wpipe->pipe_state |= PIPE_WANTW; 760 error = ltsleep(wpipe, PSOCK | PCATCH, "pipdwc", 0, 761 &wpipe->pipe_slock); 762 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 763 error = EPIPE; 764 } 765 766 /* Pipe is drained; next read will off the direct buffer */ 767 wpipe->pipe_state |= PIPE_DIRECTR; 768 769 /* Wait until the reader is done */ 770 while (error == 0 && (wpipe->pipe_state & PIPE_DIRECTR)) { 771 if (wpipe->pipe_state & PIPE_WANTR) { 772 wpipe->pipe_state &= ~PIPE_WANTR; 773 wakeup(wpipe); 774 } 775 pipeselwakeup(wpipe, wpipe, fp->f_data, POLL_IN); 776 error = ltsleep(wpipe, PSOCK | PCATCH, "pipdwt", 0, 777 &wpipe->pipe_slock); 778 if (error == 0 && wpipe->pipe_state & PIPE_EOF) 779 error = EPIPE; 780 } 781 782 /* Take pipe out of direct write mode */ 783 wpipe->pipe_state &= ~(PIPE_DIRECTW | PIPE_DIRECTR); 784 785 /* Acquire the pipe lock and cleanup */ 786 (void)pipelock(wpipe, 0); 787 if (pgs != NULL) { 788 pmap_kremove(wpipe->pipe_map.kva, blen); 789 uvm_unloan(pgs, npages, UVM_LOAN_TOPAGE); 790 } 791 if (error || amountpipekva > maxpipekva) 792 pipe_loan_free(wpipe); 793 794 if (error) { 795 pipeselwakeup(wpipe, wpipe, fp->f_data, POLL_ERR); 796 797 /* 798 * If nothing was read from what we offered, return error 799 * straight on. Otherwise update uio resid first. Caller 800 * will deal with the error condition, returning short 801 * write, error, or restarting the write(2) as appropriate. 802 */ 803 if (wpipe->pipe_map.cnt == bcnt) { 804 wpipe->pipe_map.cnt = 0; 805 wakeup(wpipe); 806 return (error); 807 } 808 809 bcnt -= wpipe->pipe_map.cnt; 810 } 811 812 uio->uio_resid -= bcnt; 813 /* uio_offset not updated, not set/used for write(2) */ 814 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + bcnt; 815 uio->uio_iov->iov_len -= bcnt; 816 if (uio->uio_iov->iov_len == 0) { 817 uio->uio_iov++; 818 uio->uio_iovcnt--; 819 } 820 821 wpipe->pipe_map.cnt = 0; 822 return (error); 823 } 824 #endif /* !PIPE_NODIRECT */ 825 826 static int 827 pipe_write(fp, offset, uio, cred, flags) 828 struct file *fp; 829 off_t *offset; 830 struct uio *uio; 831 struct ucred *cred; 832 int flags; 833 { 834 struct pipe *wpipe, *rpipe; 835 struct pipebuf *bp; 836 int error; 837 838 /* We want to write to our peer */ 839 rpipe = (struct pipe *) fp->f_data; 840 841 retry: 842 error = 0; 843 PIPE_LOCK(rpipe); 844 wpipe = rpipe->pipe_peer; 845 846 /* 847 * Detect loss of pipe read side, issue SIGPIPE if lost. 848 */ 849 if (wpipe == NULL) 850 error = EPIPE; 851 else if (simple_lock_try(&wpipe->pipe_slock) == 0) { 852 /* Deal with race for peer */ 853 PIPE_UNLOCK(rpipe); 854 goto retry; 855 } else if ((wpipe->pipe_state & PIPE_EOF) != 0) { 856 PIPE_UNLOCK(wpipe); 857 error = EPIPE; 858 } 859 860 PIPE_UNLOCK(rpipe); 861 if (error != 0) 862 return (error); 863 864 ++wpipe->pipe_busy; 865 866 /* Aquire the long-term pipe lock */ 867 if ((error = pipelock(wpipe,1)) != 0) { 868 --wpipe->pipe_busy; 869 if (wpipe->pipe_busy == 0 870 && (wpipe->pipe_state & PIPE_WANTCLOSE)) { 871 wpipe->pipe_state &= ~(PIPE_WANTCLOSE | PIPE_WANTR); 872 wakeup(wpipe); 873 } 874 PIPE_UNLOCK(wpipe); 875 return (error); 876 } 877 878 bp = &wpipe->pipe_buffer; 879 880 /* 881 * If it is advantageous to resize the pipe buffer, do so. 882 */ 883 if ((uio->uio_resid > PIPE_SIZE) && 884 (nbigpipe < maxbigpipes) && 885 #ifndef PIPE_NODIRECT 886 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 887 #endif 888 (bp->size <= PIPE_SIZE) && (bp->cnt == 0)) { 889 890 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) 891 nbigpipe++; 892 } 893 894 while (uio->uio_resid) { 895 size_t space; 896 897 #ifndef PIPE_NODIRECT 898 /* 899 * Pipe buffered writes cannot be coincidental with 900 * direct writes. Also, only one direct write can be 901 * in progress at any one time. We wait until the currently 902 * executing direct write is completed before continuing. 903 * 904 * We break out if a signal occurs or the reader goes away. 905 */ 906 while (error == 0 && wpipe->pipe_state & PIPE_DIRECTW) { 907 PIPE_LOCK(wpipe); 908 if (wpipe->pipe_state & PIPE_WANTR) { 909 wpipe->pipe_state &= ~PIPE_WANTR; 910 wakeup(wpipe); 911 } 912 pipeunlock(wpipe); 913 error = ltsleep(wpipe, PSOCK | PCATCH, 914 "pipbww", 0, &wpipe->pipe_slock); 915 916 (void)pipelock(wpipe, 0); 917 if (wpipe->pipe_state & PIPE_EOF) 918 error = EPIPE; 919 } 920 if (error) 921 break; 922 923 /* 924 * If the transfer is large, we can gain performance if 925 * we do process-to-process copies directly. 926 * If the write is non-blocking, we don't use the 927 * direct write mechanism. 928 * 929 * The direct write mechanism will detect the reader going 930 * away on us. 931 */ 932 if ((uio->uio_iov->iov_len >= PIPE_MINDIRECT) && 933 (fp->f_flag & FNONBLOCK) == 0 && 934 (wpipe->pipe_map.kva || (amountpipekva < limitpipekva))) { 935 error = pipe_direct_write(fp, wpipe, uio); 936 937 /* 938 * Break out if error occurred, unless it's ENOMEM. 939 * ENOMEM means we failed to allocate some resources 940 * for direct write, so we just fallback to ordinary 941 * write. If the direct write was successful, 942 * process rest of data via ordinary write. 943 */ 944 if (error == 0) 945 continue; 946 947 if (error != ENOMEM) 948 break; 949 } 950 #endif /* PIPE_NODIRECT */ 951 952 space = bp->size - bp->cnt; 953 954 /* Writes of size <= PIPE_BUF must be atomic. */ 955 if ((space < uio->uio_resid) && (uio->uio_resid <= PIPE_BUF)) 956 space = 0; 957 958 if (space > 0) { 959 int size; /* Transfer size */ 960 int segsize; /* first segment to transfer */ 961 962 /* 963 * Transfer size is minimum of uio transfer 964 * and free space in pipe buffer. 965 */ 966 if (space > uio->uio_resid) 967 size = uio->uio_resid; 968 else 969 size = space; 970 /* 971 * First segment to transfer is minimum of 972 * transfer size and contiguous space in 973 * pipe buffer. If first segment to transfer 974 * is less than the transfer size, we've got 975 * a wraparound in the buffer. 976 */ 977 segsize = bp->size - bp->in; 978 if (segsize > size) 979 segsize = size; 980 981 /* Transfer first segment */ 982 error = uiomove(&bp->buffer[bp->in], segsize, uio); 983 984 if (error == 0 && segsize < size) { 985 /* 986 * Transfer remaining part now, to 987 * support atomic writes. Wraparound 988 * happened. 989 */ 990 #ifdef DEBUG 991 if (bp->in + segsize != bp->size) 992 panic("Expected pipe buffer wraparound disappeared"); 993 #endif 994 995 error = uiomove(&bp->buffer[0], 996 size - segsize, uio); 997 } 998 if (error) 999 break; 1000 1001 bp->in += size; 1002 if (bp->in >= bp->size) { 1003 #ifdef DEBUG 1004 if (bp->in != size - segsize + bp->size) 1005 panic("Expected wraparound bad"); 1006 #endif 1007 bp->in = size - segsize; 1008 } 1009 1010 bp->cnt += size; 1011 #ifdef DEBUG 1012 if (bp->cnt > bp->size) 1013 panic("Pipe buffer overflow"); 1014 #endif 1015 } else { 1016 /* 1017 * If the "read-side" has been blocked, wake it up now. 1018 */ 1019 PIPE_LOCK(wpipe); 1020 if (wpipe->pipe_state & PIPE_WANTR) { 1021 wpipe->pipe_state &= ~PIPE_WANTR; 1022 wakeup(wpipe); 1023 } 1024 PIPE_UNLOCK(wpipe); 1025 1026 /* 1027 * don't block on non-blocking I/O 1028 */ 1029 if (fp->f_flag & FNONBLOCK) { 1030 error = EAGAIN; 1031 break; 1032 } 1033 1034 /* 1035 * We have no more space and have something to offer, 1036 * wake up select/poll. 1037 */ 1038 if (bp->cnt) 1039 pipeselwakeup(wpipe, wpipe, fp->f_data, 1040 POLL_OUT); 1041 1042 PIPE_LOCK(wpipe); 1043 pipeunlock(wpipe); 1044 wpipe->pipe_state |= PIPE_WANTW; 1045 error = ltsleep(wpipe, PSOCK | PCATCH, "pipewr", 0, 1046 &wpipe->pipe_slock); 1047 (void)pipelock(wpipe, 0); 1048 if (error != 0) 1049 break; 1050 /* 1051 * If read side wants to go away, we just issue a signal 1052 * to ourselves. 1053 */ 1054 if (wpipe->pipe_state & PIPE_EOF) { 1055 error = EPIPE; 1056 break; 1057 } 1058 } 1059 } 1060 1061 PIPE_LOCK(wpipe); 1062 --wpipe->pipe_busy; 1063 if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANTCLOSE)) { 1064 wpipe->pipe_state &= ~(PIPE_WANTCLOSE | PIPE_WANTR); 1065 wakeup(wpipe); 1066 } else if (bp->cnt > 0) { 1067 /* 1068 * If we have put any characters in the buffer, we wake up 1069 * the reader. 1070 */ 1071 if (wpipe->pipe_state & PIPE_WANTR) { 1072 wpipe->pipe_state &= ~PIPE_WANTR; 1073 wakeup(wpipe); 1074 } 1075 } 1076 1077 /* 1078 * Don't return EPIPE if I/O was successful 1079 */ 1080 if (error == EPIPE && bp->cnt == 0 && uio->uio_resid == 0) 1081 error = 0; 1082 1083 if (error == 0) 1084 PIPE_TIMESTAMP(&wpipe->pipe_mtime); 1085 1086 /* 1087 * We have something to offer, wake up select/poll. 1088 * wpipe->pipe_map.cnt is always 0 in this point (direct write 1089 * is only done synchronously), so check only wpipe->pipe_buffer.cnt 1090 */ 1091 if (bp->cnt) 1092 pipeselwakeup(wpipe, wpipe, fp->f_data, POLL_OUT); 1093 1094 /* 1095 * Arrange for next read(2) to do a signal. 1096 */ 1097 wpipe->pipe_state |= PIPE_SIGNALR; 1098 1099 pipeunlock(wpipe); 1100 PIPE_UNLOCK(wpipe); 1101 return (error); 1102 } 1103 1104 /* 1105 * we implement a very minimal set of ioctls for compatibility with sockets. 1106 */ 1107 int 1108 pipe_ioctl(fp, cmd, data, p) 1109 struct file *fp; 1110 u_long cmd; 1111 void *data; 1112 struct proc *p; 1113 { 1114 struct pipe *pipe = (struct pipe *)fp->f_data; 1115 1116 switch (cmd) { 1117 1118 case FIONBIO: 1119 return (0); 1120 1121 case FIOASYNC: 1122 PIPE_LOCK(pipe); 1123 if (*(int *)data) { 1124 pipe->pipe_state |= PIPE_ASYNC; 1125 } else { 1126 pipe->pipe_state &= ~PIPE_ASYNC; 1127 } 1128 PIPE_UNLOCK(pipe); 1129 return (0); 1130 1131 case FIONREAD: 1132 PIPE_LOCK(pipe); 1133 #ifndef PIPE_NODIRECT 1134 if (pipe->pipe_state & PIPE_DIRECTW) 1135 *(int *)data = pipe->pipe_map.cnt; 1136 else 1137 #endif 1138 *(int *)data = pipe->pipe_buffer.cnt; 1139 PIPE_UNLOCK(pipe); 1140 return (0); 1141 1142 case FIONWRITE: 1143 /* Look at other side */ 1144 pipe = pipe->pipe_peer; 1145 PIPE_LOCK(pipe); 1146 #ifndef PIPE_NODIRECT 1147 if (pipe->pipe_state & PIPE_DIRECTW) 1148 *(int *)data = pipe->pipe_map.cnt; 1149 else 1150 #endif 1151 *(int *)data = pipe->pipe_buffer.cnt; 1152 PIPE_UNLOCK(pipe); 1153 return (0); 1154 1155 case FIONSPACE: 1156 /* Look at other side */ 1157 pipe = pipe->pipe_peer; 1158 PIPE_LOCK(pipe); 1159 #ifndef PIPE_NODIRECT 1160 /* 1161 * If we're in direct-mode, we don't really have a 1162 * send queue, and any other write will block. Thus 1163 * zero seems like the best answer. 1164 */ 1165 if (pipe->pipe_state & PIPE_DIRECTW) 1166 *(int *)data = 0; 1167 else 1168 #endif 1169 *(int *)data = pipe->pipe_buffer.size - 1170 pipe->pipe_buffer.cnt; 1171 PIPE_UNLOCK(pipe); 1172 return (0); 1173 1174 case TIOCSPGRP: 1175 case FIOSETOWN: 1176 return fsetown(p, &pipe->pipe_pgid, cmd, data); 1177 1178 case TIOCGPGRP: 1179 case FIOGETOWN: 1180 return fgetown(p, pipe->pipe_pgid, cmd, data); 1181 1182 } 1183 return (EPASSTHROUGH); 1184 } 1185 1186 int 1187 pipe_poll(fp, events, td) 1188 struct file *fp; 1189 int events; 1190 struct proc *td; 1191 { 1192 struct pipe *rpipe = (struct pipe *)fp->f_data; 1193 struct pipe *wpipe; 1194 int eof = 0; 1195 int revents = 0; 1196 1197 retry: 1198 PIPE_LOCK(rpipe); 1199 wpipe = rpipe->pipe_peer; 1200 if (wpipe != NULL && simple_lock_try(&wpipe->pipe_slock) == 0) { 1201 /* Deal with race for peer */ 1202 PIPE_UNLOCK(rpipe); 1203 goto retry; 1204 } 1205 1206 if (events & (POLLIN | POLLRDNORM)) 1207 if ((rpipe->pipe_buffer.cnt > 0) || 1208 #ifndef PIPE_NODIRECT 1209 (rpipe->pipe_state & PIPE_DIRECTR) || 1210 #endif 1211 (rpipe->pipe_state & PIPE_EOF)) 1212 revents |= events & (POLLIN | POLLRDNORM); 1213 1214 eof |= (rpipe->pipe_state & PIPE_EOF); 1215 PIPE_UNLOCK(rpipe); 1216 1217 if (wpipe == NULL) 1218 revents |= events & (POLLOUT | POLLWRNORM); 1219 else { 1220 if (events & (POLLOUT | POLLWRNORM)) 1221 if ((wpipe->pipe_state & PIPE_EOF) || ( 1222 #ifndef PIPE_NODIRECT 1223 (wpipe->pipe_state & PIPE_DIRECTW) == 0 && 1224 #endif 1225 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 1226 revents |= events & (POLLOUT | POLLWRNORM); 1227 1228 eof |= (wpipe->pipe_state & PIPE_EOF); 1229 PIPE_UNLOCK(wpipe); 1230 } 1231 1232 if (wpipe == NULL || eof) 1233 revents |= POLLHUP; 1234 1235 if (revents == 0) { 1236 if (events & (POLLIN | POLLRDNORM)) 1237 selrecord(td, &rpipe->pipe_sel); 1238 1239 if (events & (POLLOUT | POLLWRNORM)) 1240 selrecord(td, &wpipe->pipe_sel); 1241 } 1242 1243 return (revents); 1244 } 1245 1246 static int 1247 pipe_stat(fp, ub, td) 1248 struct file *fp; 1249 struct stat *ub; 1250 struct proc *td; 1251 { 1252 struct pipe *pipe = (struct pipe *)fp->f_data; 1253 1254 memset((caddr_t)ub, 0, sizeof(*ub)); 1255 ub->st_mode = S_IFIFO | S_IRUSR | S_IWUSR; 1256 ub->st_blksize = pipe->pipe_buffer.size; 1257 ub->st_size = pipe->pipe_buffer.cnt; 1258 ub->st_blocks = (ub->st_size) ? 1 : 0; 1259 TIMEVAL_TO_TIMESPEC(&pipe->pipe_atime, &ub->st_atimespec); 1260 TIMEVAL_TO_TIMESPEC(&pipe->pipe_mtime, &ub->st_mtimespec); 1261 TIMEVAL_TO_TIMESPEC(&pipe->pipe_ctime, &ub->st_ctimespec); 1262 ub->st_uid = fp->f_cred->cr_uid; 1263 ub->st_gid = fp->f_cred->cr_gid; 1264 /* 1265 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 1266 * XXX (st_dev, st_ino) should be unique. 1267 */ 1268 return (0); 1269 } 1270 1271 /* ARGSUSED */ 1272 static int 1273 pipe_close(fp, td) 1274 struct file *fp; 1275 struct proc *td; 1276 { 1277 struct pipe *pipe = (struct pipe *)fp->f_data; 1278 1279 fp->f_data = NULL; 1280 pipeclose(fp, pipe); 1281 return (0); 1282 } 1283 1284 static void 1285 pipe_free_kmem(pipe) 1286 struct pipe *pipe; 1287 { 1288 1289 if (pipe->pipe_buffer.buffer != NULL) { 1290 if (pipe->pipe_buffer.size > PIPE_SIZE) 1291 --nbigpipe; 1292 amountpipekva -= pipe->pipe_buffer.size; 1293 uvm_km_free(kernel_map, 1294 (vaddr_t)pipe->pipe_buffer.buffer, 1295 pipe->pipe_buffer.size); 1296 pipe->pipe_buffer.buffer = NULL; 1297 } 1298 #ifndef PIPE_NODIRECT 1299 if (pipe->pipe_map.kva != 0) { 1300 pipe_loan_free(pipe); 1301 pipe->pipe_map.cnt = 0; 1302 pipe->pipe_map.kva = 0; 1303 pipe->pipe_map.pos = 0; 1304 pipe->pipe_map.npages = 0; 1305 } 1306 #endif /* !PIPE_NODIRECT */ 1307 } 1308 1309 /* 1310 * shutdown the pipe 1311 */ 1312 static void 1313 pipeclose(fp, pipe) 1314 struct file *fp; 1315 struct pipe *pipe; 1316 { 1317 struct pipe *ppipe; 1318 1319 if (pipe == NULL) 1320 return; 1321 1322 retry: 1323 PIPE_LOCK(pipe); 1324 1325 if (fp) 1326 pipeselwakeup(pipe, pipe, fp->f_data, POLL_HUP); 1327 1328 /* 1329 * If the other side is blocked, wake it up saying that 1330 * we want to close it down. 1331 */ 1332 while (pipe->pipe_busy) { 1333 wakeup(pipe); 1334 pipe->pipe_state |= PIPE_WANTCLOSE | PIPE_EOF; 1335 ltsleep(pipe, PSOCK, "pipecl", 0, &pipe->pipe_slock); 1336 } 1337 1338 /* 1339 * Disconnect from peer 1340 */ 1341 if ((ppipe = pipe->pipe_peer) != NULL) { 1342 /* Deal with race for peer */ 1343 if (simple_lock_try(&ppipe->pipe_slock) == 0) { 1344 PIPE_UNLOCK(pipe); 1345 goto retry; 1346 } 1347 if (fp) 1348 pipeselwakeup(ppipe, ppipe, fp->f_data, POLL_HUP); 1349 1350 ppipe->pipe_state |= PIPE_EOF; 1351 wakeup(ppipe); 1352 ppipe->pipe_peer = NULL; 1353 PIPE_UNLOCK(ppipe); 1354 } 1355 1356 (void)lockmgr(&pipe->pipe_lock, LK_DRAIN | LK_INTERLOCK, 1357 &pipe->pipe_slock); 1358 1359 /* 1360 * free resources 1361 */ 1362 pipe_free_kmem(pipe); 1363 pool_put(&pipe_pool, pipe); 1364 } 1365 1366 static void 1367 filt_pipedetach(struct knote *kn) 1368 { 1369 struct pipe *pipe = (struct pipe *)kn->kn_fp->f_data; 1370 1371 switch(kn->kn_filter) { 1372 case EVFILT_WRITE: 1373 /* need the peer structure, not our own */ 1374 pipe = pipe->pipe_peer; 1375 /* XXXSMP: race for peer */ 1376 1377 /* if reader end already closed, just return */ 1378 if (pipe == NULL) 1379 return; 1380 1381 break; 1382 default: 1383 /* nothing to do */ 1384 break; 1385 } 1386 1387 #ifdef DIAGNOSTIC 1388 if (kn->kn_hook != pipe) 1389 panic("filt_pipedetach: inconsistent knote"); 1390 #endif 1391 1392 PIPE_LOCK(pipe); 1393 SLIST_REMOVE(&pipe->pipe_sel.sel_klist, kn, knote, kn_selnext); 1394 PIPE_UNLOCK(pipe); 1395 } 1396 1397 /*ARGSUSED*/ 1398 static int 1399 filt_piperead(struct knote *kn, long hint) 1400 { 1401 struct pipe *rpipe = (struct pipe *)kn->kn_fp->f_data; 1402 struct pipe *wpipe = rpipe->pipe_peer; 1403 1404 if ((hint & NOTE_SUBMIT) == 0) 1405 PIPE_LOCK(rpipe); 1406 kn->kn_data = rpipe->pipe_buffer.cnt; 1407 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1408 kn->kn_data = rpipe->pipe_map.cnt; 1409 1410 /* XXXSMP: race for peer */ 1411 if ((rpipe->pipe_state & PIPE_EOF) || 1412 (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1413 kn->kn_flags |= EV_EOF; 1414 if ((hint & NOTE_SUBMIT) == 0) 1415 PIPE_UNLOCK(rpipe); 1416 return (1); 1417 } 1418 if ((hint & NOTE_SUBMIT) == 0) 1419 PIPE_UNLOCK(rpipe); 1420 return (kn->kn_data > 0); 1421 } 1422 1423 /*ARGSUSED*/ 1424 static int 1425 filt_pipewrite(struct knote *kn, long hint) 1426 { 1427 struct pipe *rpipe = (struct pipe *)kn->kn_fp->f_data; 1428 struct pipe *wpipe = rpipe->pipe_peer; 1429 1430 if ((hint & NOTE_SUBMIT) == 0) 1431 PIPE_LOCK(rpipe); 1432 /* XXXSMP: race for peer */ 1433 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) { 1434 kn->kn_data = 0; 1435 kn->kn_flags |= EV_EOF; 1436 if ((hint & NOTE_SUBMIT) == 0) 1437 PIPE_UNLOCK(rpipe); 1438 return (1); 1439 } 1440 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1441 if (wpipe->pipe_state & PIPE_DIRECTW) 1442 kn->kn_data = 0; 1443 1444 if ((hint & NOTE_SUBMIT) == 0) 1445 PIPE_UNLOCK(rpipe); 1446 return (kn->kn_data >= PIPE_BUF); 1447 } 1448 1449 static const struct filterops pipe_rfiltops = 1450 { 1, NULL, filt_pipedetach, filt_piperead }; 1451 static const struct filterops pipe_wfiltops = 1452 { 1, NULL, filt_pipedetach, filt_pipewrite }; 1453 1454 /*ARGSUSED*/ 1455 static int 1456 pipe_kqfilter(struct file *fp, struct knote *kn) 1457 { 1458 struct pipe *pipe; 1459 1460 pipe = (struct pipe *)kn->kn_fp->f_data; 1461 switch (kn->kn_filter) { 1462 case EVFILT_READ: 1463 kn->kn_fop = &pipe_rfiltops; 1464 break; 1465 case EVFILT_WRITE: 1466 kn->kn_fop = &pipe_wfiltops; 1467 /* XXXSMP: race for peer */ 1468 pipe = pipe->pipe_peer; 1469 if (pipe == NULL) { 1470 /* other end of pipe has been closed */ 1471 return (EBADF); 1472 } 1473 break; 1474 default: 1475 return (1); 1476 } 1477 kn->kn_hook = pipe; 1478 1479 PIPE_LOCK(pipe); 1480 SLIST_INSERT_HEAD(&pipe->pipe_sel.sel_klist, kn, kn_selnext); 1481 PIPE_UNLOCK(pipe); 1482 return (0); 1483 } 1484 1485 /* 1486 * Handle pipe sysctls. 1487 */ 1488 SYSCTL_SETUP(sysctl_kern_pipe_setup, "sysctl kern.pipe subtree setup") 1489 { 1490 1491 sysctl_createv(clog, 0, NULL, NULL, 1492 CTLFLAG_PERMANENT, 1493 CTLTYPE_NODE, "kern", NULL, 1494 NULL, 0, NULL, 0, 1495 CTL_KERN, CTL_EOL); 1496 sysctl_createv(clog, 0, NULL, NULL, 1497 CTLFLAG_PERMANENT, 1498 CTLTYPE_NODE, "pipe", 1499 SYSCTL_DESCR("Pipe settings"), 1500 NULL, 0, NULL, 0, 1501 CTL_KERN, KERN_PIPE, CTL_EOL); 1502 1503 sysctl_createv(clog, 0, NULL, NULL, 1504 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1505 CTLTYPE_INT, "maxkvasz", 1506 SYSCTL_DESCR("Maximum amount of kernel memory to be " 1507 "used for pipes"), 1508 NULL, 0, &maxpipekva, 0, 1509 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXKVASZ, CTL_EOL); 1510 sysctl_createv(clog, 0, NULL, NULL, 1511 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1512 CTLTYPE_INT, "maxloankvasz", 1513 SYSCTL_DESCR("Limit for direct transfers via page loan"), 1514 NULL, 0, &limitpipekva, 0, 1515 CTL_KERN, KERN_PIPE, KERN_PIPE_LIMITKVA, CTL_EOL); 1516 sysctl_createv(clog, 0, NULL, NULL, 1517 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1518 CTLTYPE_INT, "maxbigpipes", 1519 SYSCTL_DESCR("Maximum number of \"big\" pipes"), 1520 NULL, 0, &maxbigpipes, 0, 1521 CTL_KERN, KERN_PIPE, KERN_PIPE_MAXBIGPIPES, CTL_EOL); 1522 sysctl_createv(clog, 0, NULL, NULL, 1523 CTLFLAG_PERMANENT, 1524 CTLTYPE_INT, "nbigpipes", 1525 SYSCTL_DESCR("Number of \"big\" pipes"), 1526 NULL, 0, &nbigpipe, 0, 1527 CTL_KERN, KERN_PIPE, KERN_PIPE_NBIGPIPES, CTL_EOL); 1528 sysctl_createv(clog, 0, NULL, NULL, 1529 CTLFLAG_PERMANENT, 1530 CTLTYPE_INT, "kvasize", 1531 SYSCTL_DESCR("Amount of kernel memory consumed by pipe " 1532 "buffers"), 1533 NULL, 0, &amountpipekva, 0, 1534 CTL_KERN, KERN_PIPE, KERN_PIPE_KVASIZE, CTL_EOL); 1535 } 1536