1 /* $OpenBSD: uipc_socket.c,v 1.343 2024/08/11 00:19:00 jsg Exp $ */ 2 /* $NetBSD: uipc_socket.c,v 1.21 1996/02/04 02:17:52 christos Exp $ */ 3 4 /* 5 * Copyright (c) 1982, 1986, 1988, 1990, 1993 6 * The Regents of the University of California. All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of the University nor the names of its contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * @(#)uipc_socket.c 8.3 (Berkeley) 4/15/94 33 */ 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/proc.h> 38 #include <sys/file.h> 39 #include <sys/filedesc.h> 40 #include <sys/malloc.h> 41 #include <sys/mbuf.h> 42 #include <sys/domain.h> 43 #include <sys/event.h> 44 #include <sys/protosw.h> 45 #include <sys/socket.h> 46 #include <sys/unpcb.h> 47 #include <sys/socketvar.h> 48 #include <sys/signalvar.h> 49 #include <sys/pool.h> 50 #include <sys/atomic.h> 51 #include <sys/rwlock.h> 52 #include <sys/time.h> 53 #include <sys/refcnt.h> 54 55 #ifdef DDB 56 #include <machine/db_machdep.h> 57 #endif 58 59 void sbsync(struct sockbuf *, struct mbuf *); 60 61 int sosplice(struct socket *, int, off_t, struct timeval *); 62 void sounsplice(struct socket *, struct socket *, int); 63 void soidle(void *); 64 void sotask(void *); 65 void soreaper(void *); 66 void soput(void *); 67 int somove(struct socket *, int); 68 void sorflush(struct socket *); 69 70 void filt_sordetach(struct knote *kn); 71 int filt_soread(struct knote *kn, long hint); 72 void filt_sowdetach(struct knote *kn); 73 int filt_sowrite(struct knote *kn, long hint); 74 int filt_soexcept(struct knote *kn, long hint); 75 76 int filt_sowmodify(struct kevent *kev, struct knote *kn); 77 int filt_sowprocess(struct knote *kn, struct kevent *kev); 78 79 int filt_sormodify(struct kevent *kev, struct knote *kn); 80 int filt_sorprocess(struct knote *kn, struct kevent *kev); 81 82 const struct filterops soread_filtops = { 83 .f_flags = FILTEROP_ISFD | FILTEROP_MPSAFE, 84 .f_attach = NULL, 85 .f_detach = filt_sordetach, 86 .f_event = filt_soread, 87 .f_modify = filt_sormodify, 88 .f_process = filt_sorprocess, 89 }; 90 91 const struct filterops sowrite_filtops = { 92 .f_flags = FILTEROP_ISFD | FILTEROP_MPSAFE, 93 .f_attach = NULL, 94 .f_detach = filt_sowdetach, 95 .f_event = filt_sowrite, 96 .f_modify = filt_sowmodify, 97 .f_process = filt_sowprocess, 98 }; 99 100 const struct filterops soexcept_filtops = { 101 .f_flags = FILTEROP_ISFD | FILTEROP_MPSAFE, 102 .f_attach = NULL, 103 .f_detach = filt_sordetach, 104 .f_event = filt_soexcept, 105 .f_modify = filt_sormodify, 106 .f_process = filt_sorprocess, 107 }; 108 109 #ifndef SOMINCONN 110 #define SOMINCONN 80 111 #endif /* SOMINCONN */ 112 113 int somaxconn = SOMAXCONN; 114 int sominconn = SOMINCONN; 115 116 struct pool socket_pool; 117 #ifdef SOCKET_SPLICE 118 struct pool sosplice_pool; 119 struct taskq *sosplice_taskq; 120 struct rwlock sosplice_lock = RWLOCK_INITIALIZER("sosplicelk"); 121 #endif 122 123 void 124 soinit(void) 125 { 126 pool_init(&socket_pool, sizeof(struct socket), 0, IPL_SOFTNET, 0, 127 "sockpl", NULL); 128 #ifdef SOCKET_SPLICE 129 pool_init(&sosplice_pool, sizeof(struct sosplice), 0, IPL_SOFTNET, 0, 130 "sosppl", NULL); 131 #endif 132 } 133 134 struct socket * 135 soalloc(const struct protosw *prp, int wait) 136 { 137 const struct domain *dp = prp->pr_domain; 138 struct socket *so; 139 140 so = pool_get(&socket_pool, (wait == M_WAIT ? PR_WAITOK : PR_NOWAIT) | 141 PR_ZERO); 142 if (so == NULL) 143 return (NULL); 144 rw_init_flags(&so->so_lock, dp->dom_name, RWL_DUPOK); 145 refcnt_init(&so->so_refcnt); 146 rw_init(&so->so_rcv.sb_lock, "sbufrcv"); 147 rw_init(&so->so_snd.sb_lock, "sbufsnd"); 148 mtx_init_flags(&so->so_rcv.sb_mtx, IPL_MPFLOOR, "sbrcv", 0); 149 mtx_init_flags(&so->so_snd.sb_mtx, IPL_MPFLOOR, "sbsnd", 0); 150 klist_init_mutex(&so->so_rcv.sb_klist, &so->so_rcv.sb_mtx); 151 klist_init_mutex(&so->so_snd.sb_klist, &so->so_snd.sb_mtx); 152 sigio_init(&so->so_sigio); 153 TAILQ_INIT(&so->so_q0); 154 TAILQ_INIT(&so->so_q); 155 156 switch (dp->dom_family) { 157 case AF_INET: 158 case AF_INET6: 159 switch (prp->pr_type) { 160 case SOCK_RAW: 161 case SOCK_DGRAM: 162 so->so_snd.sb_flags |= SB_MTXLOCK; 163 so->so_rcv.sb_flags |= SB_MTXLOCK; 164 break; 165 } 166 break; 167 case AF_KEY: 168 case AF_ROUTE: 169 case AF_UNIX: 170 so->so_snd.sb_flags |= SB_MTXLOCK; 171 so->so_rcv.sb_flags |= SB_MTXLOCK; 172 break; 173 } 174 175 return (so); 176 } 177 178 /* 179 * Socket operation routines. 180 * These routines are called by the routines in 181 * sys_socket.c or from a system process, and 182 * implement the semantics of socket operations by 183 * switching out to the protocol specific routines. 184 */ 185 int 186 socreate(int dom, struct socket **aso, int type, int proto) 187 { 188 struct proc *p = curproc; /* XXX */ 189 const struct protosw *prp; 190 struct socket *so; 191 int error; 192 193 if (proto) 194 prp = pffindproto(dom, proto, type); 195 else 196 prp = pffindtype(dom, type); 197 if (prp == NULL || prp->pr_usrreqs == NULL) 198 return (EPROTONOSUPPORT); 199 if (prp->pr_type != type) 200 return (EPROTOTYPE); 201 so = soalloc(prp, M_WAIT); 202 so->so_type = type; 203 if (suser(p) == 0) 204 so->so_state = SS_PRIV; 205 so->so_ruid = p->p_ucred->cr_ruid; 206 so->so_euid = p->p_ucred->cr_uid; 207 so->so_rgid = p->p_ucred->cr_rgid; 208 so->so_egid = p->p_ucred->cr_gid; 209 so->so_cpid = p->p_p->ps_pid; 210 so->so_proto = prp; 211 so->so_snd.sb_timeo_nsecs = INFSLP; 212 so->so_rcv.sb_timeo_nsecs = INFSLP; 213 214 solock(so); 215 error = pru_attach(so, proto, M_WAIT); 216 if (error) { 217 so->so_state |= SS_NOFDREF; 218 /* sofree() calls sounlock(). */ 219 sofree(so, 0); 220 return (error); 221 } 222 sounlock(so); 223 *aso = so; 224 return (0); 225 } 226 227 int 228 sobind(struct socket *so, struct mbuf *nam, struct proc *p) 229 { 230 soassertlocked(so); 231 return pru_bind(so, nam, p); 232 } 233 234 int 235 solisten(struct socket *so, int backlog) 236 { 237 int somaxconn_local = atomic_load_int(&somaxconn); 238 int sominconn_local = atomic_load_int(&sominconn); 239 int error; 240 241 switch (so->so_type) { 242 case SOCK_STREAM: 243 case SOCK_SEQPACKET: 244 break; 245 default: 246 return (EOPNOTSUPP); 247 } 248 249 soassertlocked(so); 250 251 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING|SS_ISDISCONNECTING)) 252 return (EINVAL); 253 #ifdef SOCKET_SPLICE 254 if (isspliced(so) || issplicedback(so)) 255 return (EOPNOTSUPP); 256 #endif /* SOCKET_SPLICE */ 257 error = pru_listen(so); 258 if (error) 259 return (error); 260 if (TAILQ_FIRST(&so->so_q) == NULL) 261 so->so_options |= SO_ACCEPTCONN; 262 if (backlog < 0 || backlog > somaxconn_local) 263 backlog = somaxconn_local; 264 if (backlog < sominconn_local) 265 backlog = sominconn_local; 266 so->so_qlimit = backlog; 267 return (0); 268 } 269 270 #define SOSP_FREEING_READ 1 271 #define SOSP_FREEING_WRITE 2 272 void 273 sofree(struct socket *so, int keep_lock) 274 { 275 int persocket = solock_persocket(so); 276 277 soassertlocked(so); 278 279 if (so->so_pcb || (so->so_state & SS_NOFDREF) == 0) { 280 if (!keep_lock) 281 sounlock(so); 282 return; 283 } 284 if (so->so_head) { 285 struct socket *head = so->so_head; 286 287 /* 288 * We must not decommission a socket that's on the accept(2) 289 * queue. If we do, then accept(2) may hang after select(2) 290 * indicated that the listening socket was ready. 291 */ 292 if (so->so_onq == &head->so_q) { 293 if (!keep_lock) 294 sounlock(so); 295 return; 296 } 297 298 if (persocket) { 299 /* 300 * Concurrent close of `head' could 301 * abort `so' due to re-lock. 302 */ 303 soref(so); 304 soref(head); 305 sounlock(so); 306 solock(head); 307 solock(so); 308 309 if (so->so_onq != &head->so_q0) { 310 sounlock(head); 311 sounlock(so); 312 sorele(head); 313 sorele(so); 314 return; 315 } 316 317 sorele(head); 318 sorele(so); 319 } 320 321 soqremque(so, 0); 322 323 if (persocket) 324 sounlock(head); 325 } 326 327 switch (so->so_proto->pr_domain->dom_family) { 328 case AF_INET: 329 case AF_INET6: 330 if (so->so_proto->pr_type == SOCK_STREAM) 331 break; 332 /* FALLTHROUGH */ 333 default: 334 sounlock(so); 335 refcnt_finalize(&so->so_refcnt, "sofinal"); 336 solock(so); 337 break; 338 } 339 340 sigio_free(&so->so_sigio); 341 klist_free(&so->so_rcv.sb_klist); 342 klist_free(&so->so_snd.sb_klist); 343 344 mtx_enter(&so->so_snd.sb_mtx); 345 sbrelease(so, &so->so_snd); 346 mtx_leave(&so->so_snd.sb_mtx); 347 348 /* 349 * Unlocked dispose and cleanup is safe. Socket is unlinked 350 * from everywhere. Even concurrent sotask() thread will not 351 * call somove(). 352 */ 353 if (so->so_proto->pr_flags & PR_RIGHTS && 354 so->so_proto->pr_domain->dom_dispose) 355 (*so->so_proto->pr_domain->dom_dispose)(so->so_rcv.sb_mb); 356 m_purge(so->so_rcv.sb_mb); 357 358 if (!keep_lock) 359 sounlock(so); 360 361 #ifdef SOCKET_SPLICE 362 if (so->so_sp) { 363 /* Reuse splice idle, sounsplice() has been called before. */ 364 timeout_set_flags(&so->so_sp->ssp_idleto, soreaper, so, 365 KCLOCK_NONE, TIMEOUT_PROC | TIMEOUT_MPSAFE); 366 timeout_add(&so->so_sp->ssp_idleto, 0); 367 } else 368 #endif /* SOCKET_SPLICE */ 369 { 370 pool_put(&socket_pool, so); 371 } 372 } 373 374 static inline uint64_t 375 solinger_nsec(struct socket *so) 376 { 377 if (so->so_linger == 0) 378 return INFSLP; 379 380 return SEC_TO_NSEC(so->so_linger); 381 } 382 383 /* 384 * Close a socket on last file table reference removal. 385 * Initiate disconnect if connected. 386 * Free socket when disconnect complete. 387 */ 388 int 389 soclose(struct socket *so, int flags) 390 { 391 struct socket *so2; 392 int error = 0; 393 394 solock(so); 395 /* Revoke async IO early. There is a final revocation in sofree(). */ 396 sigio_free(&so->so_sigio); 397 if (so->so_state & SS_ISCONNECTED) { 398 if (so->so_pcb == NULL) 399 goto discard; 400 if ((so->so_state & SS_ISDISCONNECTING) == 0) { 401 error = sodisconnect(so); 402 if (error) 403 goto drop; 404 } 405 if (so->so_options & SO_LINGER) { 406 if ((so->so_state & SS_ISDISCONNECTING) && 407 (flags & MSG_DONTWAIT)) 408 goto drop; 409 while (so->so_state & SS_ISCONNECTED) { 410 error = sosleep_nsec(so, &so->so_timeo, 411 PSOCK | PCATCH, "netcls", 412 solinger_nsec(so)); 413 if (error) 414 break; 415 } 416 } 417 } 418 drop: 419 if (so->so_pcb) { 420 int error2; 421 error2 = pru_detach(so); 422 if (error == 0) 423 error = error2; 424 } 425 if (so->so_options & SO_ACCEPTCONN) { 426 int persocket = solock_persocket(so); 427 428 while ((so2 = TAILQ_FIRST(&so->so_q0)) != NULL) { 429 if (persocket) 430 solock(so2); 431 (void) soqremque(so2, 0); 432 if (persocket) 433 sounlock(so); 434 soabort(so2); 435 if (persocket) 436 solock(so); 437 } 438 while ((so2 = TAILQ_FIRST(&so->so_q)) != NULL) { 439 if (persocket) 440 solock(so2); 441 (void) soqremque(so2, 1); 442 if (persocket) 443 sounlock(so); 444 soabort(so2); 445 if (persocket) 446 solock(so); 447 } 448 } 449 discard: 450 if (so->so_state & SS_NOFDREF) 451 panic("soclose NOFDREF: so %p, so_type %d", so, so->so_type); 452 so->so_state |= SS_NOFDREF; 453 454 #ifdef SOCKET_SPLICE 455 if (so->so_sp) { 456 struct socket *soback; 457 458 if (so->so_proto->pr_flags & PR_WANTRCVD) { 459 /* 460 * Copy - Paste, but can't relock and sleep in 461 * sofree() in tcp(4) case. That's why tcp(4) 462 * still rely on solock() for splicing and 463 * unsplicing. 464 */ 465 466 if (issplicedback(so)) { 467 int freeing = SOSP_FREEING_WRITE; 468 469 if (so->so_sp->ssp_soback == so) 470 freeing |= SOSP_FREEING_READ; 471 sounsplice(so->so_sp->ssp_soback, so, freeing); 472 } 473 if (isspliced(so)) { 474 int freeing = SOSP_FREEING_READ; 475 476 if (so == so->so_sp->ssp_socket) 477 freeing |= SOSP_FREEING_WRITE; 478 sounsplice(so, so->so_sp->ssp_socket, freeing); 479 } 480 goto free; 481 } 482 483 sounlock(so); 484 mtx_enter(&so->so_snd.sb_mtx); 485 /* 486 * Concurrent sounsplice() locks `sb_mtx' mutexes on 487 * both `so_snd' and `so_rcv' before unsplice sockets. 488 */ 489 if ((soback = so->so_sp->ssp_soback) == NULL) { 490 mtx_leave(&so->so_snd.sb_mtx); 491 goto notsplicedback; 492 } 493 soref(soback); 494 mtx_leave(&so->so_snd.sb_mtx); 495 496 /* 497 * `so' can be only unspliced, and never spliced again. 498 * Thus if issplicedback(so) check is positive, socket is 499 * still spliced and `ssp_soback' points to the same 500 * socket that `soback'. 501 */ 502 sblock(&soback->so_rcv, SBL_WAIT | SBL_NOINTR); 503 if (issplicedback(so)) { 504 int freeing = SOSP_FREEING_WRITE; 505 506 if (so->so_sp->ssp_soback == so) 507 freeing |= SOSP_FREEING_READ; 508 solock(soback); 509 sounsplice(so->so_sp->ssp_soback, so, freeing); 510 sounlock(soback); 511 } 512 sbunlock(&soback->so_rcv); 513 sorele(soback); 514 515 notsplicedback: 516 sblock(&so->so_rcv, SBL_WAIT | SBL_NOINTR); 517 if (isspliced(so)) { 518 int freeing = SOSP_FREEING_READ; 519 520 if (so == so->so_sp->ssp_socket) 521 freeing |= SOSP_FREEING_WRITE; 522 solock(so); 523 sounsplice(so, so->so_sp->ssp_socket, freeing); 524 sounlock(so); 525 } 526 sbunlock(&so->so_rcv); 527 528 solock(so); 529 } 530 free: 531 #endif /* SOCKET_SPLICE */ 532 /* sofree() calls sounlock(). */ 533 sofree(so, 0); 534 return (error); 535 } 536 537 void 538 soabort(struct socket *so) 539 { 540 soassertlocked(so); 541 pru_abort(so); 542 } 543 544 int 545 soaccept(struct socket *so, struct mbuf *nam) 546 { 547 int error = 0; 548 549 soassertlocked(so); 550 551 if ((so->so_state & SS_NOFDREF) == 0) 552 panic("soaccept !NOFDREF: so %p, so_type %d", so, so->so_type); 553 so->so_state &= ~SS_NOFDREF; 554 if ((so->so_state & SS_ISDISCONNECTED) == 0 || 555 (so->so_proto->pr_flags & PR_ABRTACPTDIS) == 0) 556 error = pru_accept(so, nam); 557 else 558 error = ECONNABORTED; 559 return (error); 560 } 561 562 int 563 soconnect(struct socket *so, struct mbuf *nam) 564 { 565 int error; 566 567 soassertlocked(so); 568 569 if (so->so_options & SO_ACCEPTCONN) 570 return (EOPNOTSUPP); 571 /* 572 * If protocol is connection-based, can only connect once. 573 * Otherwise, if connected, try to disconnect first. 574 * This allows user to disconnect by connecting to, e.g., 575 * a null address. 576 */ 577 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) && 578 ((so->so_proto->pr_flags & PR_CONNREQUIRED) || 579 (error = sodisconnect(so)))) 580 error = EISCONN; 581 else 582 error = pru_connect(so, nam); 583 return (error); 584 } 585 586 int 587 soconnect2(struct socket *so1, struct socket *so2) 588 { 589 int persocket, error; 590 591 if ((persocket = solock_persocket(so1))) 592 solock_pair(so1, so2); 593 else 594 solock(so1); 595 596 error = pru_connect2(so1, so2); 597 598 if (persocket) 599 sounlock(so2); 600 sounlock(so1); 601 return (error); 602 } 603 604 int 605 sodisconnect(struct socket *so) 606 { 607 int error; 608 609 soassertlocked(so); 610 611 if ((so->so_state & SS_ISCONNECTED) == 0) 612 return (ENOTCONN); 613 if (so->so_state & SS_ISDISCONNECTING) 614 return (EALREADY); 615 error = pru_disconnect(so); 616 return (error); 617 } 618 619 int m_getuio(struct mbuf **, int, long, struct uio *); 620 621 #define SBLOCKWAIT(f) (((f) & MSG_DONTWAIT) ? 0 : SBL_WAIT) 622 /* 623 * Send on a socket. 624 * If send must go all at once and message is larger than 625 * send buffering, then hard error. 626 * Lock against other senders. 627 * If must go all at once and not enough room now, then 628 * inform user that this would block and do nothing. 629 * Otherwise, if nonblocking, send as much as possible. 630 * The data to be sent is described by "uio" if nonzero, 631 * otherwise by the mbuf chain "top" (which must be null 632 * if uio is not). Data provided in mbuf chain must be small 633 * enough to send all at once. 634 * 635 * Returns nonzero on error, timeout or signal; callers 636 * must check for short counts if EINTR/ERESTART are returned. 637 * Data and control buffers are freed on return. 638 */ 639 int 640 sosend(struct socket *so, struct mbuf *addr, struct uio *uio, struct mbuf *top, 641 struct mbuf *control, int flags) 642 { 643 long space, clen = 0; 644 size_t resid; 645 int error; 646 int atomic = sosendallatonce(so) || top; 647 int dosolock = ((so->so_snd.sb_flags & SB_MTXLOCK) == 0); 648 649 if (uio) 650 resid = uio->uio_resid; 651 else 652 resid = top->m_pkthdr.len; 653 /* MSG_EOR on a SOCK_STREAM socket is invalid. */ 654 if (so->so_type == SOCK_STREAM && (flags & MSG_EOR)) { 655 m_freem(top); 656 m_freem(control); 657 return (EINVAL); 658 } 659 if (uio && uio->uio_procp) 660 uio->uio_procp->p_ru.ru_msgsnd++; 661 if (control) { 662 /* 663 * In theory clen should be unsigned (since control->m_len is). 664 * However, space must be signed, as it might be less than 0 665 * if we over-committed, and we must use a signed comparison 666 * of space and clen. 667 */ 668 clen = control->m_len; 669 /* reserve extra space for AF_UNIX's internalize */ 670 if (so->so_proto->pr_domain->dom_family == AF_UNIX && 671 clen >= CMSG_ALIGN(sizeof(struct cmsghdr)) && 672 mtod(control, struct cmsghdr *)->cmsg_type == SCM_RIGHTS) 673 clen = CMSG_SPACE( 674 (clen - CMSG_ALIGN(sizeof(struct cmsghdr))) * 675 (sizeof(struct fdpass) / sizeof(int))); 676 } 677 678 #define snderr(errno) { error = errno; goto release; } 679 680 restart: 681 if ((error = sblock(&so->so_snd, SBLOCKWAIT(flags))) != 0) 682 goto out; 683 if (dosolock) 684 solock_shared(so); 685 sb_mtx_lock(&so->so_snd); 686 so->so_snd.sb_state |= SS_ISSENDING; 687 do { 688 if (so->so_snd.sb_state & SS_CANTSENDMORE) 689 snderr(EPIPE); 690 if ((error = READ_ONCE(so->so_error))) { 691 so->so_error = 0; 692 snderr(error); 693 } 694 if ((so->so_state & SS_ISCONNECTED) == 0) { 695 if (so->so_proto->pr_flags & PR_CONNREQUIRED) { 696 if (!(resid == 0 && clen != 0)) 697 snderr(ENOTCONN); 698 } else if (addr == NULL) 699 snderr(EDESTADDRREQ); 700 } 701 space = sbspace_locked(so, &so->so_snd); 702 if (flags & MSG_OOB) 703 space += 1024; 704 if (so->so_proto->pr_domain->dom_family == AF_UNIX) { 705 if (atomic && resid > so->so_snd.sb_hiwat) 706 snderr(EMSGSIZE); 707 } else { 708 if (clen > so->so_snd.sb_hiwat || 709 (atomic && resid > so->so_snd.sb_hiwat - clen)) 710 snderr(EMSGSIZE); 711 } 712 if (space < clen || 713 (space - clen < resid && 714 (atomic || space < so->so_snd.sb_lowat))) { 715 if (flags & MSG_DONTWAIT) 716 snderr(EWOULDBLOCK); 717 sbunlock(&so->so_snd); 718 error = sbwait(so, &so->so_snd); 719 so->so_snd.sb_state &= ~SS_ISSENDING; 720 sb_mtx_unlock(&so->so_snd); 721 if (dosolock) 722 sounlock_shared(so); 723 if (error) 724 goto out; 725 goto restart; 726 } 727 space -= clen; 728 do { 729 if (uio == NULL) { 730 /* 731 * Data is prepackaged in "top". 732 */ 733 resid = 0; 734 if (flags & MSG_EOR) 735 top->m_flags |= M_EOR; 736 } else { 737 sb_mtx_unlock(&so->so_snd); 738 if (dosolock) 739 sounlock_shared(so); 740 error = m_getuio(&top, atomic, space, uio); 741 if (dosolock) 742 solock_shared(so); 743 sb_mtx_lock(&so->so_snd); 744 if (error) 745 goto release; 746 space -= top->m_pkthdr.len; 747 resid = uio->uio_resid; 748 if (flags & MSG_EOR) 749 top->m_flags |= M_EOR; 750 } 751 if (resid == 0) 752 so->so_snd.sb_state &= ~SS_ISSENDING; 753 if (top && so->so_options & SO_ZEROIZE) 754 top->m_flags |= M_ZEROIZE; 755 sb_mtx_unlock(&so->so_snd); 756 if (!dosolock) 757 solock_shared(so); 758 if (flags & MSG_OOB) 759 error = pru_sendoob(so, top, addr, control); 760 else 761 error = pru_send(so, top, addr, control); 762 if (!dosolock) 763 sounlock_shared(so); 764 sb_mtx_lock(&so->so_snd); 765 clen = 0; 766 control = NULL; 767 top = NULL; 768 if (error) 769 goto release; 770 } while (resid && space > 0); 771 } while (resid); 772 773 release: 774 so->so_snd.sb_state &= ~SS_ISSENDING; 775 sb_mtx_unlock(&so->so_snd); 776 if (dosolock) 777 sounlock_shared(so); 778 sbunlock(&so->so_snd); 779 out: 780 m_freem(top); 781 m_freem(control); 782 return (error); 783 } 784 785 int 786 m_getuio(struct mbuf **mp, int atomic, long space, struct uio *uio) 787 { 788 struct mbuf *m, *top = NULL; 789 struct mbuf **nextp = ⊤ 790 u_long len, mlen; 791 size_t resid = uio->uio_resid; 792 int error; 793 794 do { 795 if (top == NULL) { 796 MGETHDR(m, M_WAIT, MT_DATA); 797 mlen = MHLEN; 798 m->m_pkthdr.len = 0; 799 m->m_pkthdr.ph_ifidx = 0; 800 } else { 801 MGET(m, M_WAIT, MT_DATA); 802 mlen = MLEN; 803 } 804 /* chain mbuf together */ 805 *nextp = m; 806 nextp = &m->m_next; 807 808 resid = ulmin(resid, space); 809 if (resid >= MINCLSIZE) { 810 MCLGETL(m, M_NOWAIT, ulmin(resid, MAXMCLBYTES)); 811 if ((m->m_flags & M_EXT) == 0) 812 MCLGETL(m, M_NOWAIT, MCLBYTES); 813 if ((m->m_flags & M_EXT) == 0) 814 goto nopages; 815 mlen = m->m_ext.ext_size; 816 len = ulmin(mlen, resid); 817 /* 818 * For datagram protocols, leave room 819 * for protocol headers in first mbuf. 820 */ 821 if (atomic && m == top && len < mlen - max_hdr) 822 m->m_data += max_hdr; 823 } else { 824 nopages: 825 len = ulmin(mlen, resid); 826 /* 827 * For datagram protocols, leave room 828 * for protocol headers in first mbuf. 829 */ 830 if (atomic && m == top && len < mlen - max_hdr) 831 m_align(m, len); 832 } 833 834 error = uiomove(mtod(m, caddr_t), len, uio); 835 if (error) { 836 m_freem(top); 837 return (error); 838 } 839 840 /* adjust counters */ 841 resid = uio->uio_resid; 842 space -= len; 843 m->m_len = len; 844 top->m_pkthdr.len += len; 845 846 /* Is there more space and more data? */ 847 } while (space > 0 && resid > 0); 848 849 *mp = top; 850 return 0; 851 } 852 853 /* 854 * Following replacement or removal of the first mbuf on the first 855 * mbuf chain of a socket buffer, push necessary state changes back 856 * into the socket buffer so that other consumers see the values 857 * consistently. 'nextrecord' is the callers locally stored value of 858 * the original value of sb->sb_mb->m_nextpkt which must be restored 859 * when the lead mbuf changes. NOTE: 'nextrecord' may be NULL. 860 */ 861 void 862 sbsync(struct sockbuf *sb, struct mbuf *nextrecord) 863 { 864 865 /* 866 * First, update for the new value of nextrecord. If necessary, 867 * make it the first record. 868 */ 869 if (sb->sb_mb != NULL) 870 sb->sb_mb->m_nextpkt = nextrecord; 871 else 872 sb->sb_mb = nextrecord; 873 874 /* 875 * Now update any dependent socket buffer fields to reflect 876 * the new state. This is an inline of SB_EMPTY_FIXUP, with 877 * the addition of a second clause that takes care of the 878 * case where sb_mb has been updated, but remains the last 879 * record. 880 */ 881 if (sb->sb_mb == NULL) { 882 sb->sb_mbtail = NULL; 883 sb->sb_lastrecord = NULL; 884 } else if (sb->sb_mb->m_nextpkt == NULL) 885 sb->sb_lastrecord = sb->sb_mb; 886 } 887 888 /* 889 * Implement receive operations on a socket. 890 * We depend on the way that records are added to the sockbuf 891 * by sbappend*. In particular, each record (mbufs linked through m_next) 892 * must begin with an address if the protocol so specifies, 893 * followed by an optional mbuf or mbufs containing ancillary data, 894 * and then zero or more mbufs of data. 895 * In order to avoid blocking network for the entire time here, we release 896 * the solock() while doing the actual copy to user space. 897 * Although the sockbuf is locked, new data may still be appended, 898 * and thus we must maintain consistency of the sockbuf during that time. 899 * 900 * The caller may receive the data as a single mbuf chain by supplying 901 * an mbuf **mp0 for use in returning the chain. The uio is then used 902 * only for the count in uio_resid. 903 */ 904 int 905 soreceive(struct socket *so, struct mbuf **paddr, struct uio *uio, 906 struct mbuf **mp0, struct mbuf **controlp, int *flagsp, 907 socklen_t controllen) 908 { 909 struct mbuf *m, **mp; 910 struct mbuf *cm; 911 u_long len, offset, moff; 912 int flags, error, error2, type, uio_error = 0; 913 const struct protosw *pr = so->so_proto; 914 struct mbuf *nextrecord; 915 size_t resid, orig_resid = uio->uio_resid; 916 int dosolock = ((so->so_rcv.sb_flags & SB_MTXLOCK) == 0); 917 918 mp = mp0; 919 if (paddr) 920 *paddr = NULL; 921 if (controlp) 922 *controlp = NULL; 923 if (flagsp) 924 flags = *flagsp &~ MSG_EOR; 925 else 926 flags = 0; 927 if (flags & MSG_OOB) { 928 m = m_get(M_WAIT, MT_DATA); 929 solock(so); 930 error = pru_rcvoob(so, m, flags & MSG_PEEK); 931 sounlock(so); 932 if (error) 933 goto bad; 934 do { 935 error = uiomove(mtod(m, caddr_t), 936 ulmin(uio->uio_resid, m->m_len), uio); 937 m = m_free(m); 938 } while (uio->uio_resid && error == 0 && m); 939 bad: 940 m_freem(m); 941 return (error); 942 } 943 if (mp) 944 *mp = NULL; 945 946 restart: 947 if ((error = sblock(&so->so_rcv, SBLOCKWAIT(flags))) != 0) 948 return (error); 949 if (dosolock) 950 solock_shared(so); 951 sb_mtx_lock(&so->so_rcv); 952 953 m = so->so_rcv.sb_mb; 954 #ifdef SOCKET_SPLICE 955 if (isspliced(so)) 956 m = NULL; 957 #endif /* SOCKET_SPLICE */ 958 /* 959 * If we have less data than requested, block awaiting more 960 * (subject to any timeout) if: 961 * 1. the current count is less than the low water mark, 962 * 2. MSG_WAITALL is set, and it is possible to do the entire 963 * receive operation at once if we block (resid <= hiwat), or 964 * 3. MSG_DONTWAIT is not set. 965 * If MSG_WAITALL is set but resid is larger than the receive buffer, 966 * we have to do the receive in sections, and thus risk returning 967 * a short count if a timeout or signal occurs after we start. 968 */ 969 if (m == NULL || (((flags & MSG_DONTWAIT) == 0 && 970 so->so_rcv.sb_cc < uio->uio_resid) && 971 (so->so_rcv.sb_cc < so->so_rcv.sb_lowat || 972 ((flags & MSG_WAITALL) && uio->uio_resid <= so->so_rcv.sb_hiwat)) && 973 m->m_nextpkt == NULL && (pr->pr_flags & PR_ATOMIC) == 0)) { 974 #ifdef DIAGNOSTIC 975 if (m == NULL && so->so_rcv.sb_cc) 976 #ifdef SOCKET_SPLICE 977 if (!isspliced(so)) 978 #endif /* SOCKET_SPLICE */ 979 panic("receive 1: so %p, so_type %d, sb_cc %lu", 980 so, so->so_type, so->so_rcv.sb_cc); 981 #endif 982 if ((error2 = READ_ONCE(so->so_error))) { 983 if (m) 984 goto dontblock; 985 error = error2; 986 if ((flags & MSG_PEEK) == 0) 987 so->so_error = 0; 988 goto release; 989 } 990 if (so->so_rcv.sb_state & SS_CANTRCVMORE) { 991 if (m) 992 goto dontblock; 993 else if (so->so_rcv.sb_cc == 0) 994 goto release; 995 } 996 for (; m; m = m->m_next) 997 if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) { 998 m = so->so_rcv.sb_mb; 999 goto dontblock; 1000 } 1001 if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 && 1002 (so->so_proto->pr_flags & PR_CONNREQUIRED)) { 1003 error = ENOTCONN; 1004 goto release; 1005 } 1006 if (uio->uio_resid == 0 && controlp == NULL) 1007 goto release; 1008 if (flags & MSG_DONTWAIT) { 1009 error = EWOULDBLOCK; 1010 goto release; 1011 } 1012 SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 1"); 1013 SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 1"); 1014 1015 sbunlock(&so->so_rcv); 1016 error = sbwait(so, &so->so_rcv); 1017 sb_mtx_unlock(&so->so_rcv); 1018 if (dosolock) 1019 sounlock_shared(so); 1020 if (error) 1021 return (error); 1022 goto restart; 1023 } 1024 dontblock: 1025 /* 1026 * On entry here, m points to the first record of the socket buffer. 1027 * From this point onward, we maintain 'nextrecord' as a cache of the 1028 * pointer to the next record in the socket buffer. We must keep the 1029 * various socket buffer pointers and local stack versions of the 1030 * pointers in sync, pushing out modifications before operations that 1031 * may sleep, and re-reading them afterwards. 1032 * 1033 * Otherwise, we will race with the network stack appending new data 1034 * or records onto the socket buffer by using inconsistent/stale 1035 * versions of the field, possibly resulting in socket buffer 1036 * corruption. 1037 */ 1038 if (uio->uio_procp) 1039 uio->uio_procp->p_ru.ru_msgrcv++; 1040 KASSERT(m == so->so_rcv.sb_mb); 1041 SBLASTRECORDCHK(&so->so_rcv, "soreceive 1"); 1042 SBLASTMBUFCHK(&so->so_rcv, "soreceive 1"); 1043 nextrecord = m->m_nextpkt; 1044 if (pr->pr_flags & PR_ADDR) { 1045 #ifdef DIAGNOSTIC 1046 if (m->m_type != MT_SONAME) 1047 panic("receive 1a: so %p, so_type %d, m %p, m_type %d", 1048 so, so->so_type, m, m->m_type); 1049 #endif 1050 orig_resid = 0; 1051 if (flags & MSG_PEEK) { 1052 if (paddr) 1053 *paddr = m_copym(m, 0, m->m_len, M_NOWAIT); 1054 m = m->m_next; 1055 } else { 1056 sbfree(so, &so->so_rcv, m); 1057 if (paddr) { 1058 *paddr = m; 1059 so->so_rcv.sb_mb = m->m_next; 1060 m->m_next = NULL; 1061 m = so->so_rcv.sb_mb; 1062 } else { 1063 so->so_rcv.sb_mb = m_free(m); 1064 m = so->so_rcv.sb_mb; 1065 } 1066 sbsync(&so->so_rcv, nextrecord); 1067 } 1068 } 1069 while (m && m->m_type == MT_CONTROL && error == 0) { 1070 int skip = 0; 1071 if (flags & MSG_PEEK) { 1072 if (mtod(m, struct cmsghdr *)->cmsg_type == 1073 SCM_RIGHTS) { 1074 /* don't leak internalized SCM_RIGHTS msgs */ 1075 skip = 1; 1076 } else if (controlp) 1077 *controlp = m_copym(m, 0, m->m_len, M_NOWAIT); 1078 m = m->m_next; 1079 } else { 1080 sbfree(so, &so->so_rcv, m); 1081 so->so_rcv.sb_mb = m->m_next; 1082 m->m_nextpkt = m->m_next = NULL; 1083 cm = m; 1084 m = so->so_rcv.sb_mb; 1085 sbsync(&so->so_rcv, nextrecord); 1086 if (controlp) { 1087 if (pr->pr_domain->dom_externalize) { 1088 sb_mtx_unlock(&so->so_rcv); 1089 if (dosolock) 1090 sounlock_shared(so); 1091 error = 1092 (*pr->pr_domain->dom_externalize) 1093 (cm, controllen, flags); 1094 if (dosolock) 1095 solock_shared(so); 1096 sb_mtx_lock(&so->so_rcv); 1097 } 1098 *controlp = cm; 1099 } else { 1100 /* 1101 * Dispose of any SCM_RIGHTS message that went 1102 * through the read path rather than recv. 1103 */ 1104 if (pr->pr_domain->dom_dispose) { 1105 sb_mtx_unlock(&so->so_rcv); 1106 pr->pr_domain->dom_dispose(cm); 1107 sb_mtx_lock(&so->so_rcv); 1108 } 1109 m_free(cm); 1110 } 1111 } 1112 if (m != NULL) 1113 nextrecord = so->so_rcv.sb_mb->m_nextpkt; 1114 else 1115 nextrecord = so->so_rcv.sb_mb; 1116 if (controlp && !skip) 1117 controlp = &(*controlp)->m_next; 1118 orig_resid = 0; 1119 } 1120 1121 /* If m is non-NULL, we have some data to read. */ 1122 if (m) { 1123 type = m->m_type; 1124 if (type == MT_OOBDATA) 1125 flags |= MSG_OOB; 1126 if (m->m_flags & M_BCAST) 1127 flags |= MSG_BCAST; 1128 if (m->m_flags & M_MCAST) 1129 flags |= MSG_MCAST; 1130 } 1131 SBLASTRECORDCHK(&so->so_rcv, "soreceive 2"); 1132 SBLASTMBUFCHK(&so->so_rcv, "soreceive 2"); 1133 1134 moff = 0; 1135 offset = 0; 1136 while (m && uio->uio_resid > 0 && error == 0) { 1137 if (m->m_type == MT_OOBDATA) { 1138 if (type != MT_OOBDATA) 1139 break; 1140 } else if (type == MT_OOBDATA) { 1141 break; 1142 } else if (m->m_type == MT_CONTROL) { 1143 /* 1144 * If there is more than one control message in the 1145 * stream, we do a short read. Next can be received 1146 * or disposed by another system call. 1147 */ 1148 break; 1149 #ifdef DIAGNOSTIC 1150 } else if (m->m_type != MT_DATA && m->m_type != MT_HEADER) { 1151 panic("receive 3: so %p, so_type %d, m %p, m_type %d", 1152 so, so->so_type, m, m->m_type); 1153 #endif 1154 } 1155 so->so_rcv.sb_state &= ~SS_RCVATMARK; 1156 len = uio->uio_resid; 1157 if (so->so_oobmark && len > so->so_oobmark - offset) 1158 len = so->so_oobmark - offset; 1159 if (len > m->m_len - moff) 1160 len = m->m_len - moff; 1161 /* 1162 * If mp is set, just pass back the mbufs. 1163 * Otherwise copy them out via the uio, then free. 1164 * Sockbuf must be consistent here (points to current mbuf, 1165 * it points to next record) when we drop priority; 1166 * we must note any additions to the sockbuf when we 1167 * block interrupts again. 1168 */ 1169 if (mp == NULL && uio_error == 0) { 1170 SBLASTRECORDCHK(&so->so_rcv, "soreceive uiomove"); 1171 SBLASTMBUFCHK(&so->so_rcv, "soreceive uiomove"); 1172 resid = uio->uio_resid; 1173 sb_mtx_unlock(&so->so_rcv); 1174 if (dosolock) 1175 sounlock_shared(so); 1176 uio_error = uiomove(mtod(m, caddr_t) + moff, len, uio); 1177 if (dosolock) 1178 solock_shared(so); 1179 sb_mtx_lock(&so->so_rcv); 1180 if (uio_error) 1181 uio->uio_resid = resid - len; 1182 } else 1183 uio->uio_resid -= len; 1184 if (len == m->m_len - moff) { 1185 if (m->m_flags & M_EOR) 1186 flags |= MSG_EOR; 1187 if (flags & MSG_PEEK) { 1188 m = m->m_next; 1189 moff = 0; 1190 orig_resid = 0; 1191 } else { 1192 nextrecord = m->m_nextpkt; 1193 sbfree(so, &so->so_rcv, m); 1194 if (mp) { 1195 *mp = m; 1196 mp = &m->m_next; 1197 so->so_rcv.sb_mb = m = m->m_next; 1198 *mp = NULL; 1199 } else { 1200 so->so_rcv.sb_mb = m_free(m); 1201 m = so->so_rcv.sb_mb; 1202 } 1203 /* 1204 * If m != NULL, we also know that 1205 * so->so_rcv.sb_mb != NULL. 1206 */ 1207 KASSERT(so->so_rcv.sb_mb == m); 1208 if (m) { 1209 m->m_nextpkt = nextrecord; 1210 if (nextrecord == NULL) 1211 so->so_rcv.sb_lastrecord = m; 1212 } else { 1213 so->so_rcv.sb_mb = nextrecord; 1214 SB_EMPTY_FIXUP(&so->so_rcv); 1215 } 1216 SBLASTRECORDCHK(&so->so_rcv, "soreceive 3"); 1217 SBLASTMBUFCHK(&so->so_rcv, "soreceive 3"); 1218 } 1219 } else { 1220 if (flags & MSG_PEEK) { 1221 moff += len; 1222 orig_resid = 0; 1223 } else { 1224 if (mp) 1225 *mp = m_copym(m, 0, len, M_WAIT); 1226 m->m_data += len; 1227 m->m_len -= len; 1228 so->so_rcv.sb_cc -= len; 1229 so->so_rcv.sb_datacc -= len; 1230 } 1231 } 1232 if (so->so_oobmark) { 1233 if ((flags & MSG_PEEK) == 0) { 1234 so->so_oobmark -= len; 1235 if (so->so_oobmark == 0) { 1236 so->so_rcv.sb_state |= SS_RCVATMARK; 1237 break; 1238 } 1239 } else { 1240 offset += len; 1241 if (offset == so->so_oobmark) 1242 break; 1243 } 1244 } 1245 if (flags & MSG_EOR) 1246 break; 1247 /* 1248 * If the MSG_WAITALL flag is set (for non-atomic socket), 1249 * we must not quit until "uio->uio_resid == 0" or an error 1250 * termination. If a signal/timeout occurs, return 1251 * with a short count but without error. 1252 * Keep sockbuf locked against other readers. 1253 */ 1254 while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 && 1255 !sosendallatonce(so) && !nextrecord) { 1256 if (so->so_rcv.sb_state & SS_CANTRCVMORE || 1257 so->so_error) 1258 break; 1259 SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 2"); 1260 SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 2"); 1261 if (sbwait(so, &so->so_rcv)) { 1262 sb_mtx_unlock(&so->so_rcv); 1263 if (dosolock) 1264 sounlock_shared(so); 1265 sbunlock(&so->so_rcv); 1266 return (0); 1267 } 1268 if ((m = so->so_rcv.sb_mb) != NULL) 1269 nextrecord = m->m_nextpkt; 1270 } 1271 } 1272 1273 if (m && pr->pr_flags & PR_ATOMIC) { 1274 flags |= MSG_TRUNC; 1275 if ((flags & MSG_PEEK) == 0) 1276 (void) sbdroprecord(so, &so->so_rcv); 1277 } 1278 if ((flags & MSG_PEEK) == 0) { 1279 if (m == NULL) { 1280 /* 1281 * First part is an inline SB_EMPTY_FIXUP(). Second 1282 * part makes sure sb_lastrecord is up-to-date if 1283 * there is still data in the socket buffer. 1284 */ 1285 so->so_rcv.sb_mb = nextrecord; 1286 if (so->so_rcv.sb_mb == NULL) { 1287 so->so_rcv.sb_mbtail = NULL; 1288 so->so_rcv.sb_lastrecord = NULL; 1289 } else if (nextrecord->m_nextpkt == NULL) 1290 so->so_rcv.sb_lastrecord = nextrecord; 1291 } 1292 SBLASTRECORDCHK(&so->so_rcv, "soreceive 4"); 1293 SBLASTMBUFCHK(&so->so_rcv, "soreceive 4"); 1294 if (pr->pr_flags & PR_WANTRCVD) { 1295 sb_mtx_unlock(&so->so_rcv); 1296 if (!dosolock) 1297 solock_shared(so); 1298 pru_rcvd(so); 1299 if (!dosolock) 1300 sounlock_shared(so); 1301 sb_mtx_lock(&so->so_rcv); 1302 } 1303 } 1304 if (orig_resid == uio->uio_resid && orig_resid && 1305 (flags & MSG_EOR) == 0 && 1306 (so->so_rcv.sb_state & SS_CANTRCVMORE) == 0) { 1307 sb_mtx_unlock(&so->so_rcv); 1308 sbunlock(&so->so_rcv); 1309 goto restart; 1310 } 1311 1312 if (uio_error) 1313 error = uio_error; 1314 1315 if (flagsp) 1316 *flagsp |= flags; 1317 release: 1318 sb_mtx_unlock(&so->so_rcv); 1319 if (dosolock) 1320 sounlock_shared(so); 1321 sbunlock(&so->so_rcv); 1322 return (error); 1323 } 1324 1325 int 1326 soshutdown(struct socket *so, int how) 1327 { 1328 int error = 0; 1329 1330 switch (how) { 1331 case SHUT_RD: 1332 sorflush(so); 1333 break; 1334 case SHUT_RDWR: 1335 sorflush(so); 1336 /* FALLTHROUGH */ 1337 case SHUT_WR: 1338 solock(so); 1339 error = pru_shutdown(so); 1340 sounlock(so); 1341 break; 1342 default: 1343 error = EINVAL; 1344 break; 1345 } 1346 1347 return (error); 1348 } 1349 1350 void 1351 sorflush(struct socket *so) 1352 { 1353 struct sockbuf *sb = &so->so_rcv; 1354 struct mbuf *m; 1355 const struct protosw *pr = so->so_proto; 1356 int error; 1357 1358 error = sblock(sb, SBL_WAIT | SBL_NOINTR); 1359 /* with SBL_WAIT and SLB_NOINTR sblock() must not fail */ 1360 KASSERT(error == 0); 1361 1362 solock_shared(so); 1363 socantrcvmore(so); 1364 mtx_enter(&sb->sb_mtx); 1365 m = sb->sb_mb; 1366 memset(&sb->sb_startzero, 0, 1367 (caddr_t)&sb->sb_endzero - (caddr_t)&sb->sb_startzero); 1368 sb->sb_timeo_nsecs = INFSLP; 1369 mtx_leave(&sb->sb_mtx); 1370 sounlock_shared(so); 1371 sbunlock(sb); 1372 1373 if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose) 1374 (*pr->pr_domain->dom_dispose)(m); 1375 m_purge(m); 1376 } 1377 1378 #ifdef SOCKET_SPLICE 1379 1380 #define so_splicelen so_sp->ssp_len 1381 #define so_splicemax so_sp->ssp_max 1382 #define so_idletv so_sp->ssp_idletv 1383 #define so_idleto so_sp->ssp_idleto 1384 #define so_splicetask so_sp->ssp_task 1385 1386 int 1387 sosplice(struct socket *so, int fd, off_t max, struct timeval *tv) 1388 { 1389 struct file *fp; 1390 struct socket *sosp; 1391 struct taskq *tq; 1392 int error = 0; 1393 1394 if ((so->so_proto->pr_flags & PR_SPLICE) == 0) 1395 return (EPROTONOSUPPORT); 1396 if (max && max < 0) 1397 return (EINVAL); 1398 if (tv && (tv->tv_sec < 0 || !timerisvalid(tv))) 1399 return (EINVAL); 1400 1401 /* If no fd is given, unsplice by removing existing link. */ 1402 if (fd < 0) { 1403 if ((error = sblock(&so->so_rcv, SBL_WAIT)) != 0) 1404 return (error); 1405 solock(so); 1406 if (so->so_options & SO_ACCEPTCONN) { 1407 error = EOPNOTSUPP; 1408 goto out; 1409 } 1410 if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 && 1411 (so->so_proto->pr_flags & PR_CONNREQUIRED)) { 1412 error = ENOTCONN; 1413 goto out; 1414 } 1415 1416 if (so->so_sp && so->so_sp->ssp_socket) 1417 sounsplice(so, so->so_sp->ssp_socket, 0); 1418 out: 1419 sounlock(so); 1420 sbunlock(&so->so_rcv); 1421 return (error); 1422 } 1423 1424 if (sosplice_taskq == NULL) { 1425 rw_enter_write(&sosplice_lock); 1426 if (sosplice_taskq == NULL) { 1427 tq = taskq_create("sosplice", 1, IPL_SOFTNET, 1428 TASKQ_MPSAFE); 1429 if (tq == NULL) { 1430 rw_exit_write(&sosplice_lock); 1431 return (ENOMEM); 1432 } 1433 /* Ensure the taskq is fully visible to other CPUs. */ 1434 membar_producer(); 1435 sosplice_taskq = tq; 1436 } 1437 rw_exit_write(&sosplice_lock); 1438 } else { 1439 /* Ensure the taskq is fully visible on this CPU. */ 1440 membar_consumer(); 1441 } 1442 1443 /* Find sosp, the drain socket where data will be spliced into. */ 1444 if ((error = getsock(curproc, fd, &fp)) != 0) 1445 return (error); 1446 sosp = fp->f_data; 1447 1448 if (sosp->so_proto->pr_usrreqs->pru_send != 1449 so->so_proto->pr_usrreqs->pru_send) { 1450 error = EPROTONOSUPPORT; 1451 goto frele; 1452 } 1453 1454 if ((error = sblock(&so->so_rcv, SBL_WAIT)) != 0) 1455 goto frele; 1456 if ((error = sblock(&sosp->so_snd, SBL_WAIT)) != 0) { 1457 sbunlock(&so->so_rcv); 1458 goto frele; 1459 } 1460 solock(so); 1461 1462 if ((so->so_options & SO_ACCEPTCONN) || 1463 (sosp->so_options & SO_ACCEPTCONN)) { 1464 error = EOPNOTSUPP; 1465 goto release; 1466 } 1467 if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 && 1468 (so->so_proto->pr_flags & PR_CONNREQUIRED)) { 1469 error = ENOTCONN; 1470 goto release; 1471 } 1472 if ((sosp->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0) { 1473 error = ENOTCONN; 1474 goto release; 1475 } 1476 if (so->so_sp == NULL) 1477 so->so_sp = pool_get(&sosplice_pool, PR_WAITOK | PR_ZERO); 1478 if (sosp->so_sp == NULL) 1479 sosp->so_sp = pool_get(&sosplice_pool, PR_WAITOK | PR_ZERO); 1480 if (so->so_sp->ssp_socket || sosp->so_sp->ssp_soback) { 1481 error = EBUSY; 1482 goto release; 1483 } 1484 1485 so->so_splicelen = 0; 1486 so->so_splicemax = max; 1487 if (tv) 1488 so->so_idletv = *tv; 1489 else 1490 timerclear(&so->so_idletv); 1491 timeout_set_flags(&so->so_idleto, soidle, so, 1492 KCLOCK_NONE, TIMEOUT_PROC | TIMEOUT_MPSAFE); 1493 task_set(&so->so_splicetask, sotask, so); 1494 1495 /* 1496 * To prevent sorwakeup() calling somove() before this somove() 1497 * has finished, the socket buffers are not marked as spliced yet. 1498 */ 1499 1500 /* Splice so and sosp together. */ 1501 mtx_enter(&so->so_rcv.sb_mtx); 1502 mtx_enter(&sosp->so_snd.sb_mtx); 1503 so->so_sp->ssp_socket = sosp; 1504 sosp->so_sp->ssp_soback = so; 1505 mtx_leave(&sosp->so_snd.sb_mtx); 1506 mtx_leave(&so->so_rcv.sb_mtx); 1507 1508 if ((so->so_proto->pr_flags & PR_WANTRCVD) == 0) 1509 sounlock(so); 1510 if (somove(so, M_WAIT)) { 1511 mtx_enter(&so->so_rcv.sb_mtx); 1512 mtx_enter(&sosp->so_snd.sb_mtx); 1513 so->so_rcv.sb_flags |= SB_SPLICE; 1514 sosp->so_snd.sb_flags |= SB_SPLICE; 1515 mtx_leave(&sosp->so_snd.sb_mtx); 1516 mtx_leave(&so->so_rcv.sb_mtx); 1517 } 1518 if ((so->so_proto->pr_flags & PR_WANTRCVD) == 0) 1519 solock(so); 1520 1521 release: 1522 sounlock(so); 1523 sbunlock(&sosp->so_snd); 1524 sbunlock(&so->so_rcv); 1525 frele: 1526 FRELE(fp, curproc); 1527 1528 return (error); 1529 } 1530 1531 void 1532 sounsplice(struct socket *so, struct socket *sosp, int freeing) 1533 { 1534 if ((so->so_proto->pr_flags & PR_WANTRCVD) == 0) 1535 sbassertlocked(&so->so_rcv); 1536 soassertlocked(so); 1537 1538 task_del(sosplice_taskq, &so->so_splicetask); 1539 timeout_del(&so->so_idleto); 1540 1541 mtx_enter(&so->so_rcv.sb_mtx); 1542 mtx_enter(&sosp->so_snd.sb_mtx); 1543 so->so_rcv.sb_flags &= ~SB_SPLICE; 1544 sosp->so_snd.sb_flags &= ~SB_SPLICE; 1545 so->so_sp->ssp_socket = sosp->so_sp->ssp_soback = NULL; 1546 mtx_leave(&sosp->so_snd.sb_mtx); 1547 mtx_leave(&so->so_rcv.sb_mtx); 1548 1549 /* Do not wakeup a socket that is about to be freed. */ 1550 if ((freeing & SOSP_FREEING_READ) == 0 && soreadable(so)) 1551 sorwakeup(so); 1552 if ((freeing & SOSP_FREEING_WRITE) == 0 && sowriteable(sosp)) 1553 sowwakeup(sosp); 1554 } 1555 1556 void 1557 soidle(void *arg) 1558 { 1559 struct socket *so = arg; 1560 1561 sblock(&so->so_rcv, SBL_WAIT | SBL_NOINTR); 1562 solock(so); 1563 /* 1564 * Depending on socket type, sblock(&so->so_rcv) or solock() 1565 * is always held while modifying SB_SPLICE and 1566 * so->so_sp->ssp_socket. 1567 */ 1568 if (so->so_rcv.sb_flags & SB_SPLICE) { 1569 so->so_error = ETIMEDOUT; 1570 sounsplice(so, so->so_sp->ssp_socket, 0); 1571 } 1572 sounlock(so); 1573 sbunlock(&so->so_rcv); 1574 } 1575 1576 void 1577 sotask(void *arg) 1578 { 1579 struct socket *so = arg; 1580 int doyield = 0; 1581 int sockstream = (so->so_proto->pr_flags & PR_WANTRCVD); 1582 1583 /* 1584 * sblock() on `so_rcv' protects sockets from being unspliced 1585 * for UDP case. TCP sockets still rely on solock(). 1586 */ 1587 1588 sblock(&so->so_rcv, SBL_WAIT | SBL_NOINTR); 1589 if (sockstream) 1590 solock(so); 1591 1592 if (so->so_rcv.sb_flags & SB_SPLICE) { 1593 if (sockstream) 1594 doyield = 1; 1595 somove(so, M_DONTWAIT); 1596 } 1597 1598 if (sockstream) 1599 sounlock(so); 1600 sbunlock(&so->so_rcv); 1601 1602 if (doyield) { 1603 /* Avoid user land starvation. */ 1604 yield(); 1605 } 1606 } 1607 1608 /* 1609 * The socket splicing task or idle timeout may sleep while grabbing the net 1610 * lock. As sofree() can be called anytime, sotask() or soidle() could access 1611 * the socket memory of a freed socket after wakeup. So delay the pool_put() 1612 * after all pending socket splicing tasks or timeouts have finished. Do this 1613 * by scheduling it on the same threads. 1614 */ 1615 void 1616 soreaper(void *arg) 1617 { 1618 struct socket *so = arg; 1619 1620 /* Reuse splice task, sounsplice() has been called before. */ 1621 task_set(&so->so_sp->ssp_task, soput, so); 1622 task_add(sosplice_taskq, &so->so_sp->ssp_task); 1623 } 1624 1625 void 1626 soput(void *arg) 1627 { 1628 struct socket *so = arg; 1629 1630 pool_put(&sosplice_pool, so->so_sp); 1631 pool_put(&socket_pool, so); 1632 } 1633 1634 /* 1635 * Move data from receive buffer of spliced source socket to send 1636 * buffer of drain socket. Try to move as much as possible in one 1637 * big chunk. It is a TCP only implementation. 1638 * Return value 0 means splicing has been finished, 1 continue. 1639 */ 1640 int 1641 somove(struct socket *so, int wait) 1642 { 1643 struct socket *sosp = so->so_sp->ssp_socket; 1644 struct mbuf *m, **mp, *nextrecord; 1645 u_long len, off, oobmark; 1646 long space; 1647 int error = 0, maxreached = 0, unsplice = 0; 1648 unsigned int rcvstate; 1649 int sockdgram = ((so->so_proto->pr_flags & 1650 PR_WANTRCVD) == 0); 1651 1652 if (sockdgram) 1653 sbassertlocked(&so->so_rcv); 1654 else 1655 soassertlocked(so); 1656 1657 mtx_enter(&so->so_rcv.sb_mtx); 1658 mtx_enter(&sosp->so_snd.sb_mtx); 1659 1660 nextpkt: 1661 if ((error = READ_ONCE(so->so_error))) 1662 goto release; 1663 if (sosp->so_snd.sb_state & SS_CANTSENDMORE) { 1664 error = EPIPE; 1665 goto release; 1666 } 1667 1668 error = READ_ONCE(sosp->so_error); 1669 if (error) { 1670 if (error != ETIMEDOUT && error != EFBIG && error != ELOOP) 1671 goto release; 1672 error = 0; 1673 } 1674 if ((sosp->so_state & SS_ISCONNECTED) == 0) 1675 goto release; 1676 1677 /* Calculate how many bytes can be copied now. */ 1678 len = so->so_rcv.sb_datacc; 1679 if (so->so_splicemax) { 1680 KASSERT(so->so_splicelen < so->so_splicemax); 1681 if (so->so_splicemax <= so->so_splicelen + len) { 1682 len = so->so_splicemax - so->so_splicelen; 1683 maxreached = 1; 1684 } 1685 } 1686 space = sbspace_locked(sosp, &sosp->so_snd); 1687 if (so->so_oobmark && so->so_oobmark < len && 1688 so->so_oobmark < space + 1024) 1689 space += 1024; 1690 if (space <= 0) { 1691 maxreached = 0; 1692 goto release; 1693 } 1694 if (space < len) { 1695 maxreached = 0; 1696 if (space < sosp->so_snd.sb_lowat) 1697 goto release; 1698 len = space; 1699 } 1700 sosp->so_snd.sb_state |= SS_ISSENDING; 1701 1702 SBLASTRECORDCHK(&so->so_rcv, "somove 1"); 1703 SBLASTMBUFCHK(&so->so_rcv, "somove 1"); 1704 m = so->so_rcv.sb_mb; 1705 if (m == NULL) 1706 goto release; 1707 nextrecord = m->m_nextpkt; 1708 1709 /* Drop address and control information not used with splicing. */ 1710 if (so->so_proto->pr_flags & PR_ADDR) { 1711 #ifdef DIAGNOSTIC 1712 if (m->m_type != MT_SONAME) 1713 panic("somove soname: so %p, so_type %d, m %p, " 1714 "m_type %d", so, so->so_type, m, m->m_type); 1715 #endif 1716 m = m->m_next; 1717 } 1718 while (m && m->m_type == MT_CONTROL) 1719 m = m->m_next; 1720 if (m == NULL) { 1721 sbdroprecord(so, &so->so_rcv); 1722 if (so->so_proto->pr_flags & PR_WANTRCVD) { 1723 mtx_leave(&sosp->so_snd.sb_mtx); 1724 mtx_leave(&so->so_rcv.sb_mtx); 1725 pru_rcvd(so); 1726 mtx_enter(&so->so_rcv.sb_mtx); 1727 mtx_enter(&sosp->so_snd.sb_mtx); 1728 } 1729 goto nextpkt; 1730 } 1731 1732 /* 1733 * By splicing sockets connected to localhost, userland might create a 1734 * loop. Dissolve splicing with error if loop is detected by counter. 1735 * 1736 * If we deal with looped broadcast/multicast packet we bail out with 1737 * no error to suppress splice termination. 1738 */ 1739 if ((m->m_flags & M_PKTHDR) && 1740 ((m->m_pkthdr.ph_loopcnt++ >= M_MAXLOOP) || 1741 ((m->m_flags & M_LOOP) && (m->m_flags & (M_BCAST|M_MCAST))))) { 1742 error = ELOOP; 1743 goto release; 1744 } 1745 1746 if (so->so_proto->pr_flags & PR_ATOMIC) { 1747 if ((m->m_flags & M_PKTHDR) == 0) 1748 panic("somove !PKTHDR: so %p, so_type %d, m %p, " 1749 "m_type %d", so, so->so_type, m, m->m_type); 1750 if (sosp->so_snd.sb_hiwat < m->m_pkthdr.len) { 1751 error = EMSGSIZE; 1752 goto release; 1753 } 1754 if (len < m->m_pkthdr.len) 1755 goto release; 1756 if (m->m_pkthdr.len < len) { 1757 maxreached = 0; 1758 len = m->m_pkthdr.len; 1759 } 1760 /* 1761 * Throw away the name mbuf after it has been assured 1762 * that the whole first record can be processed. 1763 */ 1764 m = so->so_rcv.sb_mb; 1765 sbfree(so, &so->so_rcv, m); 1766 so->so_rcv.sb_mb = m_free(m); 1767 sbsync(&so->so_rcv, nextrecord); 1768 } 1769 /* 1770 * Throw away the control mbufs after it has been assured 1771 * that the whole first record can be processed. 1772 */ 1773 m = so->so_rcv.sb_mb; 1774 while (m && m->m_type == MT_CONTROL) { 1775 sbfree(so, &so->so_rcv, m); 1776 so->so_rcv.sb_mb = m_free(m); 1777 m = so->so_rcv.sb_mb; 1778 sbsync(&so->so_rcv, nextrecord); 1779 } 1780 1781 SBLASTRECORDCHK(&so->so_rcv, "somove 2"); 1782 SBLASTMBUFCHK(&so->so_rcv, "somove 2"); 1783 1784 /* Take at most len mbufs out of receive buffer. */ 1785 for (off = 0, mp = &m; off <= len && *mp; 1786 off += (*mp)->m_len, mp = &(*mp)->m_next) { 1787 u_long size = len - off; 1788 1789 #ifdef DIAGNOSTIC 1790 if ((*mp)->m_type != MT_DATA && (*mp)->m_type != MT_HEADER) 1791 panic("somove type: so %p, so_type %d, m %p, " 1792 "m_type %d", so, so->so_type, *mp, (*mp)->m_type); 1793 #endif 1794 if ((*mp)->m_len > size) { 1795 /* 1796 * Move only a partial mbuf at maximum splice length or 1797 * if the drain buffer is too small for this large mbuf. 1798 */ 1799 if (!maxreached && sosp->so_snd.sb_datacc > 0) { 1800 len -= size; 1801 break; 1802 } 1803 *mp = m_copym(so->so_rcv.sb_mb, 0, size, wait); 1804 if (*mp == NULL) { 1805 len -= size; 1806 break; 1807 } 1808 so->so_rcv.sb_mb->m_data += size; 1809 so->so_rcv.sb_mb->m_len -= size; 1810 so->so_rcv.sb_cc -= size; 1811 so->so_rcv.sb_datacc -= size; 1812 } else { 1813 *mp = so->so_rcv.sb_mb; 1814 sbfree(so, &so->so_rcv, *mp); 1815 so->so_rcv.sb_mb = (*mp)->m_next; 1816 sbsync(&so->so_rcv, nextrecord); 1817 } 1818 } 1819 *mp = NULL; 1820 1821 SBLASTRECORDCHK(&so->so_rcv, "somove 3"); 1822 SBLASTMBUFCHK(&so->so_rcv, "somove 3"); 1823 SBCHECK(so, &so->so_rcv); 1824 if (m == NULL) 1825 goto release; 1826 m->m_nextpkt = NULL; 1827 if (m->m_flags & M_PKTHDR) { 1828 m_resethdr(m); 1829 m->m_pkthdr.len = len; 1830 } 1831 1832 /* Send window update to source peer as receive buffer has changed. */ 1833 if (so->so_proto->pr_flags & PR_WANTRCVD) { 1834 mtx_leave(&sosp->so_snd.sb_mtx); 1835 mtx_leave(&so->so_rcv.sb_mtx); 1836 pru_rcvd(so); 1837 mtx_enter(&so->so_rcv.sb_mtx); 1838 mtx_enter(&sosp->so_snd.sb_mtx); 1839 } 1840 1841 /* Receive buffer did shrink by len bytes, adjust oob. */ 1842 rcvstate = so->so_rcv.sb_state; 1843 so->so_rcv.sb_state &= ~SS_RCVATMARK; 1844 oobmark = so->so_oobmark; 1845 so->so_oobmark = oobmark > len ? oobmark - len : 0; 1846 if (oobmark) { 1847 if (oobmark == len) 1848 so->so_rcv.sb_state |= SS_RCVATMARK; 1849 if (oobmark >= len) 1850 oobmark = 0; 1851 } 1852 1853 /* 1854 * Handle oob data. If any malloc fails, ignore error. 1855 * TCP urgent data is not very reliable anyway. 1856 */ 1857 while (((rcvstate & SS_RCVATMARK) || oobmark) && 1858 (so->so_options & SO_OOBINLINE)) { 1859 struct mbuf *o = NULL; 1860 1861 if (rcvstate & SS_RCVATMARK) { 1862 o = m_get(wait, MT_DATA); 1863 rcvstate &= ~SS_RCVATMARK; 1864 } else if (oobmark) { 1865 o = m_split(m, oobmark, wait); 1866 if (o) { 1867 mtx_leave(&sosp->so_snd.sb_mtx); 1868 mtx_leave(&so->so_rcv.sb_mtx); 1869 error = pru_send(sosp, m, NULL, NULL); 1870 mtx_enter(&so->so_rcv.sb_mtx); 1871 mtx_enter(&sosp->so_snd.sb_mtx); 1872 1873 if (error) { 1874 if (sosp->so_snd.sb_state & 1875 SS_CANTSENDMORE) 1876 error = EPIPE; 1877 m_freem(o); 1878 goto release; 1879 } 1880 len -= oobmark; 1881 so->so_splicelen += oobmark; 1882 m = o; 1883 o = m_get(wait, MT_DATA); 1884 } 1885 oobmark = 0; 1886 } 1887 if (o) { 1888 o->m_len = 1; 1889 *mtod(o, caddr_t) = *mtod(m, caddr_t); 1890 1891 mtx_leave(&sosp->so_snd.sb_mtx); 1892 mtx_leave(&so->so_rcv.sb_mtx); 1893 error = pru_sendoob(sosp, o, NULL, NULL); 1894 mtx_enter(&so->so_rcv.sb_mtx); 1895 mtx_enter(&sosp->so_snd.sb_mtx); 1896 1897 if (error) { 1898 if (sosp->so_snd.sb_state & SS_CANTSENDMORE) 1899 error = EPIPE; 1900 m_freem(m); 1901 goto release; 1902 } 1903 len -= 1; 1904 so->so_splicelen += 1; 1905 if (oobmark) { 1906 oobmark -= 1; 1907 if (oobmark == 0) 1908 rcvstate |= SS_RCVATMARK; 1909 } 1910 m_adj(m, 1); 1911 } 1912 } 1913 1914 /* Append all remaining data to drain socket. */ 1915 if (so->so_rcv.sb_cc == 0 || maxreached) 1916 sosp->so_snd.sb_state &= ~SS_ISSENDING; 1917 1918 mtx_leave(&sosp->so_snd.sb_mtx); 1919 mtx_leave(&so->so_rcv.sb_mtx); 1920 1921 if (sockdgram) 1922 solock_shared(sosp); 1923 error = pru_send(sosp, m, NULL, NULL); 1924 if (sockdgram) 1925 sounlock_shared(sosp); 1926 1927 mtx_enter(&so->so_rcv.sb_mtx); 1928 mtx_enter(&sosp->so_snd.sb_mtx); 1929 1930 if (error) { 1931 if (sosp->so_snd.sb_state & SS_CANTSENDMORE || 1932 sosp->so_pcb == NULL) 1933 error = EPIPE; 1934 goto release; 1935 } 1936 so->so_splicelen += len; 1937 1938 /* Move several packets if possible. */ 1939 if (!maxreached && nextrecord) 1940 goto nextpkt; 1941 1942 release: 1943 sosp->so_snd.sb_state &= ~SS_ISSENDING; 1944 1945 if (!error && maxreached && so->so_splicemax == so->so_splicelen) 1946 error = EFBIG; 1947 if (error) 1948 WRITE_ONCE(so->so_error, error); 1949 1950 if (((so->so_rcv.sb_state & SS_CANTRCVMORE) && 1951 so->so_rcv.sb_cc == 0) || 1952 (sosp->so_snd.sb_state & SS_CANTSENDMORE) || 1953 maxreached || error) 1954 unsplice = 1; 1955 1956 mtx_leave(&sosp->so_snd.sb_mtx); 1957 mtx_leave(&so->so_rcv.sb_mtx); 1958 1959 if (unsplice) { 1960 if (sockdgram) 1961 solock(so); 1962 sounsplice(so, sosp, 0); 1963 if (sockdgram) 1964 sounlock(so); 1965 1966 return (0); 1967 } 1968 if (timerisset(&so->so_idletv)) 1969 timeout_add_tv(&so->so_idleto, &so->so_idletv); 1970 return (1); 1971 } 1972 #endif /* SOCKET_SPLICE */ 1973 1974 void 1975 sorwakeup(struct socket *so) 1976 { 1977 if ((so->so_rcv.sb_flags & SB_MTXLOCK) == 0) 1978 soassertlocked_readonly(so); 1979 1980 #ifdef SOCKET_SPLICE 1981 if (so->so_proto->pr_flags & PR_SPLICE) { 1982 sb_mtx_lock(&so->so_rcv); 1983 if (so->so_rcv.sb_flags & SB_SPLICE) 1984 task_add(sosplice_taskq, &so->so_splicetask); 1985 if (isspliced(so)) { 1986 sb_mtx_unlock(&so->so_rcv); 1987 return; 1988 } 1989 sb_mtx_unlock(&so->so_rcv); 1990 } 1991 #endif 1992 sowakeup(so, &so->so_rcv); 1993 if (so->so_upcall) 1994 (*(so->so_upcall))(so, so->so_upcallarg, M_DONTWAIT); 1995 } 1996 1997 void 1998 sowwakeup(struct socket *so) 1999 { 2000 if ((so->so_snd.sb_flags & SB_MTXLOCK) == 0) 2001 soassertlocked_readonly(so); 2002 2003 #ifdef SOCKET_SPLICE 2004 if (so->so_proto->pr_flags & PR_SPLICE) { 2005 sb_mtx_lock(&so->so_snd); 2006 if (so->so_snd.sb_flags & SB_SPLICE) 2007 task_add(sosplice_taskq, 2008 &so->so_sp->ssp_soback->so_splicetask); 2009 if (issplicedback(so)) { 2010 sb_mtx_unlock(&so->so_snd); 2011 return; 2012 } 2013 sb_mtx_unlock(&so->so_snd); 2014 } 2015 #endif 2016 sowakeup(so, &so->so_snd); 2017 } 2018 2019 int 2020 sosetopt(struct socket *so, int level, int optname, struct mbuf *m) 2021 { 2022 int error = 0; 2023 2024 if (level != SOL_SOCKET) { 2025 if (so->so_proto->pr_ctloutput) { 2026 solock(so); 2027 error = (*so->so_proto->pr_ctloutput)(PRCO_SETOPT, so, 2028 level, optname, m); 2029 sounlock(so); 2030 return (error); 2031 } 2032 error = ENOPROTOOPT; 2033 } else { 2034 switch (optname) { 2035 2036 case SO_LINGER: 2037 if (m == NULL || m->m_len != sizeof (struct linger) || 2038 mtod(m, struct linger *)->l_linger < 0 || 2039 mtod(m, struct linger *)->l_linger > SHRT_MAX) 2040 return (EINVAL); 2041 2042 solock(so); 2043 so->so_linger = mtod(m, struct linger *)->l_linger; 2044 if (*mtod(m, int *)) 2045 so->so_options |= optname; 2046 else 2047 so->so_options &= ~optname; 2048 sounlock(so); 2049 2050 break; 2051 case SO_BINDANY: 2052 if ((error = suser(curproc)) != 0) /* XXX */ 2053 return (error); 2054 /* FALLTHROUGH */ 2055 2056 case SO_DEBUG: 2057 case SO_KEEPALIVE: 2058 case SO_USELOOPBACK: 2059 case SO_BROADCAST: 2060 case SO_REUSEADDR: 2061 case SO_REUSEPORT: 2062 case SO_OOBINLINE: 2063 case SO_TIMESTAMP: 2064 case SO_ZEROIZE: 2065 if (m == NULL || m->m_len < sizeof (int)) 2066 return (EINVAL); 2067 2068 solock(so); 2069 if (*mtod(m, int *)) 2070 so->so_options |= optname; 2071 else 2072 so->so_options &= ~optname; 2073 sounlock(so); 2074 2075 break; 2076 case SO_DONTROUTE: 2077 if (m == NULL || m->m_len < sizeof (int)) 2078 return (EINVAL); 2079 if (*mtod(m, int *)) 2080 error = EOPNOTSUPP; 2081 break; 2082 2083 case SO_SNDBUF: 2084 case SO_RCVBUF: 2085 case SO_SNDLOWAT: 2086 case SO_RCVLOWAT: 2087 { 2088 struct sockbuf *sb = (optname == SO_SNDBUF || 2089 optname == SO_SNDLOWAT ? 2090 &so->so_snd : &so->so_rcv); 2091 u_long cnt; 2092 2093 if (m == NULL || m->m_len < sizeof (int)) 2094 return (EINVAL); 2095 cnt = *mtod(m, int *); 2096 if ((long)cnt <= 0) 2097 cnt = 1; 2098 2099 if (((sb->sb_flags & SB_MTXLOCK) == 0)) 2100 solock(so); 2101 mtx_enter(&sb->sb_mtx); 2102 2103 switch (optname) { 2104 case SO_SNDBUF: 2105 case SO_RCVBUF: 2106 if (sb->sb_state & 2107 (SS_CANTSENDMORE | SS_CANTRCVMORE)) { 2108 error = EINVAL; 2109 break; 2110 } 2111 if (sbcheckreserve(cnt, sb->sb_wat) || 2112 sbreserve(so, sb, cnt)) { 2113 error = ENOBUFS; 2114 break; 2115 } 2116 sb->sb_wat = cnt; 2117 break; 2118 case SO_SNDLOWAT: 2119 case SO_RCVLOWAT: 2120 sb->sb_lowat = (cnt > sb->sb_hiwat) ? 2121 sb->sb_hiwat : cnt; 2122 break; 2123 } 2124 2125 mtx_leave(&sb->sb_mtx); 2126 if (((sb->sb_flags & SB_MTXLOCK) == 0)) 2127 sounlock(so); 2128 2129 break; 2130 } 2131 2132 case SO_SNDTIMEO: 2133 case SO_RCVTIMEO: 2134 { 2135 struct sockbuf *sb = (optname == SO_SNDTIMEO ? 2136 &so->so_snd : &so->so_rcv); 2137 struct timeval tv; 2138 uint64_t nsecs; 2139 2140 if (m == NULL || m->m_len < sizeof (tv)) 2141 return (EINVAL); 2142 memcpy(&tv, mtod(m, struct timeval *), sizeof tv); 2143 if (!timerisvalid(&tv)) 2144 return (EINVAL); 2145 nsecs = TIMEVAL_TO_NSEC(&tv); 2146 if (nsecs == UINT64_MAX) 2147 return (EDOM); 2148 if (nsecs == 0) 2149 nsecs = INFSLP; 2150 2151 mtx_enter(&sb->sb_mtx); 2152 sb->sb_timeo_nsecs = nsecs; 2153 mtx_leave(&sb->sb_mtx); 2154 break; 2155 } 2156 2157 case SO_RTABLE: 2158 if (so->so_proto->pr_domain && 2159 so->so_proto->pr_domain->dom_protosw && 2160 so->so_proto->pr_ctloutput) { 2161 const struct domain *dom = 2162 so->so_proto->pr_domain; 2163 2164 level = dom->dom_protosw->pr_protocol; 2165 solock(so); 2166 error = (*so->so_proto->pr_ctloutput) 2167 (PRCO_SETOPT, so, level, optname, m); 2168 sounlock(so); 2169 } else 2170 error = ENOPROTOOPT; 2171 break; 2172 #ifdef SOCKET_SPLICE 2173 case SO_SPLICE: 2174 if (m == NULL) { 2175 error = sosplice(so, -1, 0, NULL); 2176 } else if (m->m_len < sizeof(int)) { 2177 error = EINVAL; 2178 } else if (m->m_len < sizeof(struct splice)) { 2179 error = sosplice(so, *mtod(m, int *), 0, NULL); 2180 } else { 2181 error = sosplice(so, 2182 mtod(m, struct splice *)->sp_fd, 2183 mtod(m, struct splice *)->sp_max, 2184 &mtod(m, struct splice *)->sp_idle); 2185 } 2186 break; 2187 #endif /* SOCKET_SPLICE */ 2188 2189 default: 2190 error = ENOPROTOOPT; 2191 break; 2192 } 2193 } 2194 2195 return (error); 2196 } 2197 2198 int 2199 sogetopt(struct socket *so, int level, int optname, struct mbuf *m) 2200 { 2201 int error = 0; 2202 2203 if (level != SOL_SOCKET) { 2204 if (so->so_proto->pr_ctloutput) { 2205 m->m_len = 0; 2206 2207 solock(so); 2208 error = (*so->so_proto->pr_ctloutput)(PRCO_GETOPT, so, 2209 level, optname, m); 2210 sounlock(so); 2211 return (error); 2212 } else 2213 return (ENOPROTOOPT); 2214 } else { 2215 m->m_len = sizeof (int); 2216 2217 switch (optname) { 2218 2219 case SO_LINGER: 2220 m->m_len = sizeof (struct linger); 2221 solock_shared(so); 2222 mtod(m, struct linger *)->l_onoff = 2223 so->so_options & SO_LINGER; 2224 mtod(m, struct linger *)->l_linger = so->so_linger; 2225 sounlock_shared(so); 2226 break; 2227 2228 case SO_BINDANY: 2229 case SO_USELOOPBACK: 2230 case SO_DEBUG: 2231 case SO_KEEPALIVE: 2232 case SO_REUSEADDR: 2233 case SO_REUSEPORT: 2234 case SO_BROADCAST: 2235 case SO_OOBINLINE: 2236 case SO_ACCEPTCONN: 2237 case SO_TIMESTAMP: 2238 case SO_ZEROIZE: 2239 *mtod(m, int *) = so->so_options & optname; 2240 break; 2241 2242 case SO_DONTROUTE: 2243 *mtod(m, int *) = 0; 2244 break; 2245 2246 case SO_TYPE: 2247 *mtod(m, int *) = so->so_type; 2248 break; 2249 2250 case SO_ERROR: 2251 solock(so); 2252 *mtod(m, int *) = so->so_error; 2253 so->so_error = 0; 2254 sounlock(so); 2255 2256 break; 2257 2258 case SO_DOMAIN: 2259 *mtod(m, int *) = so->so_proto->pr_domain->dom_family; 2260 break; 2261 2262 case SO_PROTOCOL: 2263 *mtod(m, int *) = so->so_proto->pr_protocol; 2264 break; 2265 2266 case SO_SNDBUF: 2267 *mtod(m, int *) = so->so_snd.sb_hiwat; 2268 break; 2269 2270 case SO_RCVBUF: 2271 *mtod(m, int *) = so->so_rcv.sb_hiwat; 2272 break; 2273 2274 case SO_SNDLOWAT: 2275 *mtod(m, int *) = so->so_snd.sb_lowat; 2276 break; 2277 2278 case SO_RCVLOWAT: 2279 *mtod(m, int *) = so->so_rcv.sb_lowat; 2280 break; 2281 2282 case SO_SNDTIMEO: 2283 case SO_RCVTIMEO: 2284 { 2285 struct sockbuf *sb = (optname == SO_SNDTIMEO ? 2286 &so->so_snd : &so->so_rcv); 2287 struct timeval tv; 2288 uint64_t nsecs; 2289 2290 mtx_enter(&sb->sb_mtx); 2291 nsecs = sb->sb_timeo_nsecs; 2292 mtx_leave(&sb->sb_mtx); 2293 2294 m->m_len = sizeof(struct timeval); 2295 memset(&tv, 0, sizeof(tv)); 2296 if (nsecs != INFSLP) 2297 NSEC_TO_TIMEVAL(nsecs, &tv); 2298 memcpy(mtod(m, struct timeval *), &tv, sizeof tv); 2299 break; 2300 } 2301 2302 case SO_RTABLE: 2303 if (so->so_proto->pr_domain && 2304 so->so_proto->pr_domain->dom_protosw && 2305 so->so_proto->pr_ctloutput) { 2306 const struct domain *dom = 2307 so->so_proto->pr_domain; 2308 2309 level = dom->dom_protosw->pr_protocol; 2310 solock(so); 2311 error = (*so->so_proto->pr_ctloutput) 2312 (PRCO_GETOPT, so, level, optname, m); 2313 sounlock(so); 2314 if (error) 2315 return (error); 2316 break; 2317 } 2318 return (ENOPROTOOPT); 2319 2320 #ifdef SOCKET_SPLICE 2321 case SO_SPLICE: 2322 { 2323 off_t len; 2324 2325 m->m_len = sizeof(off_t); 2326 solock_shared(so); 2327 len = so->so_sp ? so->so_sp->ssp_len : 0; 2328 sounlock_shared(so); 2329 memcpy(mtod(m, off_t *), &len, sizeof(off_t)); 2330 break; 2331 } 2332 #endif /* SOCKET_SPLICE */ 2333 2334 case SO_PEERCRED: 2335 if (so->so_proto->pr_protocol == AF_UNIX) { 2336 struct unpcb *unp = sotounpcb(so); 2337 2338 solock(so); 2339 if (unp->unp_flags & UNP_FEIDS) { 2340 m->m_len = sizeof(unp->unp_connid); 2341 memcpy(mtod(m, caddr_t), 2342 &(unp->unp_connid), m->m_len); 2343 sounlock(so); 2344 break; 2345 } 2346 sounlock(so); 2347 2348 return (ENOTCONN); 2349 } 2350 return (EOPNOTSUPP); 2351 2352 default: 2353 return (ENOPROTOOPT); 2354 } 2355 return (0); 2356 } 2357 } 2358 2359 void 2360 sohasoutofband(struct socket *so) 2361 { 2362 pgsigio(&so->so_sigio, SIGURG, 0); 2363 knote(&so->so_rcv.sb_klist, 0); 2364 } 2365 2366 void 2367 sofilt_lock(struct socket *so, struct sockbuf *sb) 2368 { 2369 switch (so->so_proto->pr_domain->dom_family) { 2370 case PF_INET: 2371 case PF_INET6: 2372 NET_LOCK_SHARED(); 2373 break; 2374 default: 2375 rw_enter_write(&so->so_lock); 2376 break; 2377 } 2378 2379 mtx_enter(&sb->sb_mtx); 2380 } 2381 2382 void 2383 sofilt_unlock(struct socket *so, struct sockbuf *sb) 2384 { 2385 mtx_leave(&sb->sb_mtx); 2386 2387 switch (so->so_proto->pr_domain->dom_family) { 2388 case PF_INET: 2389 case PF_INET6: 2390 NET_UNLOCK_SHARED(); 2391 break; 2392 default: 2393 rw_exit_write(&so->so_lock); 2394 break; 2395 } 2396 } 2397 2398 int 2399 soo_kqfilter(struct file *fp, struct knote *kn) 2400 { 2401 struct socket *so = kn->kn_fp->f_data; 2402 struct sockbuf *sb; 2403 2404 switch (kn->kn_filter) { 2405 case EVFILT_READ: 2406 kn->kn_fop = &soread_filtops; 2407 sb = &so->so_rcv; 2408 break; 2409 case EVFILT_WRITE: 2410 kn->kn_fop = &sowrite_filtops; 2411 sb = &so->so_snd; 2412 break; 2413 case EVFILT_EXCEPT: 2414 kn->kn_fop = &soexcept_filtops; 2415 sb = &so->so_rcv; 2416 break; 2417 default: 2418 return (EINVAL); 2419 } 2420 2421 klist_insert(&sb->sb_klist, kn); 2422 2423 return (0); 2424 } 2425 2426 void 2427 filt_sordetach(struct knote *kn) 2428 { 2429 struct socket *so = kn->kn_fp->f_data; 2430 2431 klist_remove(&so->so_rcv.sb_klist, kn); 2432 } 2433 2434 int 2435 filt_soread(struct knote *kn, long hint) 2436 { 2437 struct socket *so = kn->kn_fp->f_data; 2438 int rv = 0; 2439 2440 MUTEX_ASSERT_LOCKED(&so->so_rcv.sb_mtx); 2441 if ((so->so_rcv.sb_flags & SB_MTXLOCK) == 0) 2442 soassertlocked_readonly(so); 2443 2444 if (so->so_options & SO_ACCEPTCONN) { 2445 if (so->so_rcv.sb_flags & SB_MTXLOCK) 2446 soassertlocked_readonly(so); 2447 2448 kn->kn_data = so->so_qlen; 2449 rv = (kn->kn_data != 0); 2450 2451 if (kn->kn_flags & (__EV_POLL | __EV_SELECT)) { 2452 if (so->so_state & SS_ISDISCONNECTED) { 2453 kn->kn_flags |= __EV_HUP; 2454 rv = 1; 2455 } else { 2456 rv = soreadable(so); 2457 } 2458 } 2459 2460 return rv; 2461 } 2462 2463 kn->kn_data = so->so_rcv.sb_cc; 2464 #ifdef SOCKET_SPLICE 2465 if (isspliced(so)) { 2466 rv = 0; 2467 } else 2468 #endif /* SOCKET_SPLICE */ 2469 if (so->so_rcv.sb_state & SS_CANTRCVMORE) { 2470 kn->kn_flags |= EV_EOF; 2471 if (kn->kn_flags & __EV_POLL) { 2472 if (so->so_state & SS_ISDISCONNECTED) 2473 kn->kn_flags |= __EV_HUP; 2474 } 2475 kn->kn_fflags = so->so_error; 2476 rv = 1; 2477 } else if (so->so_error) { 2478 rv = 1; 2479 } else if (kn->kn_sfflags & NOTE_LOWAT) { 2480 rv = (kn->kn_data >= kn->kn_sdata); 2481 } else { 2482 rv = (kn->kn_data >= so->so_rcv.sb_lowat); 2483 } 2484 2485 return rv; 2486 } 2487 2488 void 2489 filt_sowdetach(struct knote *kn) 2490 { 2491 struct socket *so = kn->kn_fp->f_data; 2492 2493 klist_remove(&so->so_snd.sb_klist, kn); 2494 } 2495 2496 int 2497 filt_sowrite(struct knote *kn, long hint) 2498 { 2499 struct socket *so = kn->kn_fp->f_data; 2500 int rv; 2501 2502 MUTEX_ASSERT_LOCKED(&so->so_snd.sb_mtx); 2503 if ((so->so_snd.sb_flags & SB_MTXLOCK) == 0) 2504 soassertlocked_readonly(so); 2505 2506 kn->kn_data = sbspace_locked(so, &so->so_snd); 2507 if (so->so_snd.sb_state & SS_CANTSENDMORE) { 2508 kn->kn_flags |= EV_EOF; 2509 if (kn->kn_flags & __EV_POLL) { 2510 if (so->so_state & SS_ISDISCONNECTED) 2511 kn->kn_flags |= __EV_HUP; 2512 } 2513 kn->kn_fflags = so->so_error; 2514 rv = 1; 2515 } else if (so->so_error) { 2516 rv = 1; 2517 } else if (((so->so_state & SS_ISCONNECTED) == 0) && 2518 (so->so_proto->pr_flags & PR_CONNREQUIRED)) { 2519 rv = 0; 2520 } else if (kn->kn_sfflags & NOTE_LOWAT) { 2521 rv = (kn->kn_data >= kn->kn_sdata); 2522 } else { 2523 rv = (kn->kn_data >= so->so_snd.sb_lowat); 2524 } 2525 2526 return (rv); 2527 } 2528 2529 int 2530 filt_soexcept(struct knote *kn, long hint) 2531 { 2532 struct socket *so = kn->kn_fp->f_data; 2533 int rv = 0; 2534 2535 MUTEX_ASSERT_LOCKED(&so->so_rcv.sb_mtx); 2536 if ((so->so_rcv.sb_flags & SB_MTXLOCK) == 0) 2537 soassertlocked_readonly(so); 2538 2539 #ifdef SOCKET_SPLICE 2540 if (isspliced(so)) { 2541 rv = 0; 2542 } else 2543 #endif /* SOCKET_SPLICE */ 2544 if (kn->kn_sfflags & NOTE_OOB) { 2545 if (so->so_oobmark || (so->so_rcv.sb_state & SS_RCVATMARK)) { 2546 kn->kn_fflags |= NOTE_OOB; 2547 kn->kn_data -= so->so_oobmark; 2548 rv = 1; 2549 } 2550 } 2551 2552 if (kn->kn_flags & __EV_POLL) { 2553 if (so->so_state & SS_ISDISCONNECTED) { 2554 kn->kn_flags |= __EV_HUP; 2555 rv = 1; 2556 } 2557 } 2558 2559 return rv; 2560 } 2561 2562 int 2563 filt_sowmodify(struct kevent *kev, struct knote *kn) 2564 { 2565 struct socket *so = kn->kn_fp->f_data; 2566 int rv; 2567 2568 sofilt_lock(so, &so->so_snd); 2569 rv = knote_modify(kev, kn); 2570 sofilt_unlock(so, &so->so_snd); 2571 2572 return (rv); 2573 } 2574 2575 int 2576 filt_sowprocess(struct knote *kn, struct kevent *kev) 2577 { 2578 struct socket *so = kn->kn_fp->f_data; 2579 int rv; 2580 2581 sofilt_lock(so, &so->so_snd); 2582 rv = knote_process(kn, kev); 2583 sofilt_unlock(so, &so->so_snd); 2584 2585 return (rv); 2586 } 2587 2588 int 2589 filt_sormodify(struct kevent *kev, struct knote *kn) 2590 { 2591 struct socket *so = kn->kn_fp->f_data; 2592 int rv; 2593 2594 sofilt_lock(so, &so->so_rcv); 2595 rv = knote_modify(kev, kn); 2596 sofilt_unlock(so, &so->so_rcv); 2597 2598 return (rv); 2599 } 2600 2601 int 2602 filt_sorprocess(struct knote *kn, struct kevent *kev) 2603 { 2604 struct socket *so = kn->kn_fp->f_data; 2605 int rv; 2606 2607 sofilt_lock(so, &so->so_rcv); 2608 rv = knote_process(kn, kev); 2609 sofilt_unlock(so, &so->so_rcv); 2610 2611 return (rv); 2612 } 2613 2614 #ifdef DDB 2615 void 2616 sobuf_print(struct sockbuf *, 2617 int (*)(const char *, ...) __attribute__((__format__(__kprintf__,1,2)))); 2618 2619 void 2620 sobuf_print(struct sockbuf *sb, 2621 int (*pr)(const char *, ...) __attribute__((__format__(__kprintf__,1,2)))) 2622 { 2623 (*pr)("\tsb_cc: %lu\n", sb->sb_cc); 2624 (*pr)("\tsb_datacc: %lu\n", sb->sb_datacc); 2625 (*pr)("\tsb_hiwat: %lu\n", sb->sb_hiwat); 2626 (*pr)("\tsb_wat: %lu\n", sb->sb_wat); 2627 (*pr)("\tsb_mbcnt: %lu\n", sb->sb_mbcnt); 2628 (*pr)("\tsb_mbmax: %lu\n", sb->sb_mbmax); 2629 (*pr)("\tsb_lowat: %ld\n", sb->sb_lowat); 2630 (*pr)("\tsb_mb: %p\n", sb->sb_mb); 2631 (*pr)("\tsb_mbtail: %p\n", sb->sb_mbtail); 2632 (*pr)("\tsb_lastrecord: %p\n", sb->sb_lastrecord); 2633 (*pr)("\tsb_flags: %04x\n", sb->sb_flags); 2634 (*pr)("\tsb_state: %04x\n", sb->sb_state); 2635 (*pr)("\tsb_timeo_nsecs: %llu\n", sb->sb_timeo_nsecs); 2636 } 2637 2638 void 2639 so_print(void *v, 2640 int (*pr)(const char *, ...) __attribute__((__format__(__kprintf__,1,2)))) 2641 { 2642 struct socket *so = v; 2643 2644 (*pr)("socket %p\n", so); 2645 (*pr)("so_type: %i\n", so->so_type); 2646 (*pr)("so_options: 0x%04x\n", so->so_options); /* %b */ 2647 (*pr)("so_linger: %i\n", so->so_linger); 2648 (*pr)("so_state: 0x%04x\n", so->so_state); 2649 (*pr)("so_pcb: %p\n", so->so_pcb); 2650 (*pr)("so_proto: %p\n", so->so_proto); 2651 (*pr)("so_sigio: %p\n", so->so_sigio.sir_sigio); 2652 2653 (*pr)("so_head: %p\n", so->so_head); 2654 (*pr)("so_onq: %p\n", so->so_onq); 2655 (*pr)("so_q0: @%p first: %p\n", &so->so_q0, TAILQ_FIRST(&so->so_q0)); 2656 (*pr)("so_q: @%p first: %p\n", &so->so_q, TAILQ_FIRST(&so->so_q)); 2657 (*pr)("so_eq: next: %p\n", TAILQ_NEXT(so, so_qe)); 2658 (*pr)("so_q0len: %i\n", so->so_q0len); 2659 (*pr)("so_qlen: %i\n", so->so_qlen); 2660 (*pr)("so_qlimit: %i\n", so->so_qlimit); 2661 (*pr)("so_timeo: %i\n", so->so_timeo); 2662 (*pr)("so_obmark: %lu\n", so->so_oobmark); 2663 2664 (*pr)("so_sp: %p\n", so->so_sp); 2665 if (so->so_sp != NULL) { 2666 (*pr)("\tssp_socket: %p\n", so->so_sp->ssp_socket); 2667 (*pr)("\tssp_soback: %p\n", so->so_sp->ssp_soback); 2668 (*pr)("\tssp_len: %lld\n", 2669 (unsigned long long)so->so_sp->ssp_len); 2670 (*pr)("\tssp_max: %lld\n", 2671 (unsigned long long)so->so_sp->ssp_max); 2672 (*pr)("\tssp_idletv: %lld %ld\n", so->so_sp->ssp_idletv.tv_sec, 2673 so->so_sp->ssp_idletv.tv_usec); 2674 (*pr)("\tssp_idleto: %spending (@%i)\n", 2675 timeout_pending(&so->so_sp->ssp_idleto) ? "" : "not ", 2676 so->so_sp->ssp_idleto.to_time); 2677 } 2678 2679 (*pr)("so_rcv:\n"); 2680 sobuf_print(&so->so_rcv, pr); 2681 (*pr)("so_snd:\n"); 2682 sobuf_print(&so->so_snd, pr); 2683 2684 (*pr)("so_upcall: %p so_upcallarg: %p\n", 2685 so->so_upcall, so->so_upcallarg); 2686 2687 (*pr)("so_euid: %d so_ruid: %d\n", so->so_euid, so->so_ruid); 2688 (*pr)("so_egid: %d so_rgid: %d\n", so->so_egid, so->so_rgid); 2689 (*pr)("so_cpid: %d\n", so->so_cpid); 2690 } 2691 #endif 2692