xref: /netbsd-src/sys/kern/uipc_socket.c (revision ce099b40997c43048fb78bd578195f81d2456523)
1 /*	$NetBSD: uipc_socket.c,v 1.162 2008/04/28 20:24:05 martin Exp $	*/
2 
3 /*-
4  * Copyright (c) 2002, 2007, 2008 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Jason R. Thorpe of Wasabi Systems, Inc.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 /*
33  * Copyright (c) 2004 The FreeBSD Foundation
34  * Copyright (c) 2004 Robert Watson
35  * Copyright (c) 1982, 1986, 1988, 1990, 1993
36  *	The Regents of the University of California.  All rights reserved.
37  *
38  * Redistribution and use in source and binary forms, with or without
39  * modification, are permitted provided that the following conditions
40  * are met:
41  * 1. Redistributions of source code must retain the above copyright
42  *    notice, this list of conditions and the following disclaimer.
43  * 2. Redistributions in binary form must reproduce the above copyright
44  *    notice, this list of conditions and the following disclaimer in the
45  *    documentation and/or other materials provided with the distribution.
46  * 3. Neither the name of the University nor the names of its contributors
47  *    may be used to endorse or promote products derived from this software
48  *    without specific prior written permission.
49  *
50  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
51  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
52  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
53  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
54  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
55  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
56  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
57  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
58  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
59  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
60  * SUCH DAMAGE.
61  *
62  *	@(#)uipc_socket.c	8.6 (Berkeley) 5/2/95
63  */
64 
65 #include <sys/cdefs.h>
66 __KERNEL_RCSID(0, "$NetBSD: uipc_socket.c,v 1.162 2008/04/28 20:24:05 martin Exp $");
67 
68 #include "opt_sock_counters.h"
69 #include "opt_sosend_loan.h"
70 #include "opt_mbuftrace.h"
71 #include "opt_somaxkva.h"
72 
73 #include <sys/param.h>
74 #include <sys/systm.h>
75 #include <sys/proc.h>
76 #include <sys/file.h>
77 #include <sys/filedesc.h>
78 #include <sys/malloc.h>
79 #include <sys/mbuf.h>
80 #include <sys/domain.h>
81 #include <sys/kernel.h>
82 #include <sys/protosw.h>
83 #include <sys/socket.h>
84 #include <sys/socketvar.h>
85 #include <sys/signalvar.h>
86 #include <sys/resourcevar.h>
87 #include <sys/event.h>
88 #include <sys/poll.h>
89 #include <sys/kauth.h>
90 #include <sys/mutex.h>
91 #include <sys/condvar.h>
92 
93 #include <uvm/uvm.h>
94 
95 MALLOC_DEFINE(M_SOOPTS, "soopts", "socket options");
96 MALLOC_DEFINE(M_SONAME, "soname", "socket name");
97 
98 extern const struct fileops socketops;
99 
100 extern int	somaxconn;			/* patchable (XXX sysctl) */
101 int		somaxconn = SOMAXCONN;
102 kmutex_t	*softnet_lock;
103 
104 #ifdef SOSEND_COUNTERS
105 #include <sys/device.h>
106 
107 static struct evcnt sosend_loan_big = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
108     NULL, "sosend", "loan big");
109 static struct evcnt sosend_copy_big = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
110     NULL, "sosend", "copy big");
111 static struct evcnt sosend_copy_small = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
112     NULL, "sosend", "copy small");
113 static struct evcnt sosend_kvalimit = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
114     NULL, "sosend", "kva limit");
115 
116 #define	SOSEND_COUNTER_INCR(ev)		(ev)->ev_count++
117 
118 EVCNT_ATTACH_STATIC(sosend_loan_big);
119 EVCNT_ATTACH_STATIC(sosend_copy_big);
120 EVCNT_ATTACH_STATIC(sosend_copy_small);
121 EVCNT_ATTACH_STATIC(sosend_kvalimit);
122 #else
123 
124 #define	SOSEND_COUNTER_INCR(ev)		/* nothing */
125 
126 #endif /* SOSEND_COUNTERS */
127 
128 static struct callback_entry sokva_reclaimerentry;
129 
130 #ifdef SOSEND_NO_LOAN
131 int sock_loan_thresh = -1;
132 #else
133 int sock_loan_thresh = 4096;
134 #endif
135 
136 static kmutex_t so_pendfree_lock;
137 static struct mbuf *so_pendfree;
138 
139 #ifndef SOMAXKVA
140 #define	SOMAXKVA (16 * 1024 * 1024)
141 #endif
142 int somaxkva = SOMAXKVA;
143 static int socurkva;
144 static kcondvar_t socurkva_cv;
145 
146 #define	SOCK_LOAN_CHUNK		65536
147 
148 static size_t sodopendfree(void);
149 static size_t sodopendfreel(void);
150 
151 static vsize_t
152 sokvareserve(struct socket *so, vsize_t len)
153 {
154 	int error;
155 
156 	mutex_enter(&so_pendfree_lock);
157 	while (socurkva + len > somaxkva) {
158 		size_t freed;
159 
160 		/*
161 		 * try to do pendfree.
162 		 */
163 
164 		freed = sodopendfreel();
165 
166 		/*
167 		 * if some kva was freed, try again.
168 		 */
169 
170 		if (freed)
171 			continue;
172 
173 		SOSEND_COUNTER_INCR(&sosend_kvalimit);
174 		error = cv_wait_sig(&socurkva_cv, &so_pendfree_lock);
175 		if (error) {
176 			len = 0;
177 			break;
178 		}
179 	}
180 	socurkva += len;
181 	mutex_exit(&so_pendfree_lock);
182 	return len;
183 }
184 
185 static void
186 sokvaunreserve(vsize_t len)
187 {
188 
189 	mutex_enter(&so_pendfree_lock);
190 	socurkva -= len;
191 	cv_broadcast(&socurkva_cv);
192 	mutex_exit(&so_pendfree_lock);
193 }
194 
195 /*
196  * sokvaalloc: allocate kva for loan.
197  */
198 
199 vaddr_t
200 sokvaalloc(vsize_t len, struct socket *so)
201 {
202 	vaddr_t lva;
203 
204 	/*
205 	 * reserve kva.
206 	 */
207 
208 	if (sokvareserve(so, len) == 0)
209 		return 0;
210 
211 	/*
212 	 * allocate kva.
213 	 */
214 
215 	lva = uvm_km_alloc(kernel_map, len, 0, UVM_KMF_VAONLY | UVM_KMF_WAITVA);
216 	if (lva == 0) {
217 		sokvaunreserve(len);
218 		return (0);
219 	}
220 
221 	return lva;
222 }
223 
224 /*
225  * sokvafree: free kva for loan.
226  */
227 
228 void
229 sokvafree(vaddr_t sva, vsize_t len)
230 {
231 
232 	/*
233 	 * free kva.
234 	 */
235 
236 	uvm_km_free(kernel_map, sva, len, UVM_KMF_VAONLY);
237 
238 	/*
239 	 * unreserve kva.
240 	 */
241 
242 	sokvaunreserve(len);
243 }
244 
245 static void
246 sodoloanfree(struct vm_page **pgs, void *buf, size_t size)
247 {
248 	vaddr_t sva, eva;
249 	vsize_t len;
250 	int npgs;
251 
252 	KASSERT(pgs != NULL);
253 
254 	eva = round_page((vaddr_t) buf + size);
255 	sva = trunc_page((vaddr_t) buf);
256 	len = eva - sva;
257 	npgs = len >> PAGE_SHIFT;
258 
259 	pmap_kremove(sva, len);
260 	pmap_update(pmap_kernel());
261 	uvm_unloan(pgs, npgs, UVM_LOAN_TOPAGE);
262 	sokvafree(sva, len);
263 }
264 
265 static size_t
266 sodopendfree(void)
267 {
268 	size_t rv;
269 
270 	if (__predict_true(so_pendfree == NULL))
271 		return 0;
272 
273 	mutex_enter(&so_pendfree_lock);
274 	rv = sodopendfreel();
275 	mutex_exit(&so_pendfree_lock);
276 
277 	return rv;
278 }
279 
280 /*
281  * sodopendfreel: free mbufs on "pendfree" list.
282  * unlock and relock so_pendfree_lock when freeing mbufs.
283  *
284  * => called with so_pendfree_lock held.
285  */
286 
287 static size_t
288 sodopendfreel(void)
289 {
290 	struct mbuf *m, *next;
291 	size_t rv = 0;
292 
293 	KASSERT(mutex_owned(&so_pendfree_lock));
294 
295 	while (so_pendfree != NULL) {
296 		m = so_pendfree;
297 		so_pendfree = NULL;
298 		mutex_exit(&so_pendfree_lock);
299 
300 		for (; m != NULL; m = next) {
301 			next = m->m_next;
302 			KASSERT((~m->m_flags & (M_EXT|M_EXT_PAGES)) == 0);
303 			KASSERT(m->m_ext.ext_refcnt == 0);
304 
305 			rv += m->m_ext.ext_size;
306 			sodoloanfree(m->m_ext.ext_pgs, m->m_ext.ext_buf,
307 			    m->m_ext.ext_size);
308 			pool_cache_put(mb_cache, m);
309 		}
310 
311 		mutex_enter(&so_pendfree_lock);
312 	}
313 
314 	return (rv);
315 }
316 
317 void
318 soloanfree(struct mbuf *m, void *buf, size_t size, void *arg)
319 {
320 
321 	KASSERT(m != NULL);
322 
323 	/*
324 	 * postpone freeing mbuf.
325 	 *
326 	 * we can't do it in interrupt context
327 	 * because we need to put kva back to kernel_map.
328 	 */
329 
330 	mutex_enter(&so_pendfree_lock);
331 	m->m_next = so_pendfree;
332 	so_pendfree = m;
333 	cv_broadcast(&socurkva_cv);
334 	mutex_exit(&so_pendfree_lock);
335 }
336 
337 static long
338 sosend_loan(struct socket *so, struct uio *uio, struct mbuf *m, long space)
339 {
340 	struct iovec *iov = uio->uio_iov;
341 	vaddr_t sva, eva;
342 	vsize_t len;
343 	vaddr_t lva;
344 	int npgs, error;
345 	vaddr_t va;
346 	int i;
347 
348 	if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace))
349 		return (0);
350 
351 	if (iov->iov_len < (size_t) space)
352 		space = iov->iov_len;
353 	if (space > SOCK_LOAN_CHUNK)
354 		space = SOCK_LOAN_CHUNK;
355 
356 	eva = round_page((vaddr_t) iov->iov_base + space);
357 	sva = trunc_page((vaddr_t) iov->iov_base);
358 	len = eva - sva;
359 	npgs = len >> PAGE_SHIFT;
360 
361 	KASSERT(npgs <= M_EXT_MAXPAGES);
362 
363 	lva = sokvaalloc(len, so);
364 	if (lva == 0)
365 		return 0;
366 
367 	error = uvm_loan(&uio->uio_vmspace->vm_map, sva, len,
368 	    m->m_ext.ext_pgs, UVM_LOAN_TOPAGE);
369 	if (error) {
370 		sokvafree(lva, len);
371 		return (0);
372 	}
373 
374 	for (i = 0, va = lva; i < npgs; i++, va += PAGE_SIZE)
375 		pmap_kenter_pa(va, VM_PAGE_TO_PHYS(m->m_ext.ext_pgs[i]),
376 		    VM_PROT_READ);
377 	pmap_update(pmap_kernel());
378 
379 	lva += (vaddr_t) iov->iov_base & PAGE_MASK;
380 
381 	MEXTADD(m, (void *) lva, space, M_MBUF, soloanfree, so);
382 	m->m_flags |= M_EXT_PAGES | M_EXT_ROMAP;
383 
384 	uio->uio_resid -= space;
385 	/* uio_offset not updated, not set/used for write(2) */
386 	uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + space;
387 	uio->uio_iov->iov_len -= space;
388 	if (uio->uio_iov->iov_len == 0) {
389 		uio->uio_iov++;
390 		uio->uio_iovcnt--;
391 	}
392 
393 	return (space);
394 }
395 
396 static int
397 sokva_reclaim_callback(struct callback_entry *ce, void *obj, void *arg)
398 {
399 
400 	KASSERT(ce == &sokva_reclaimerentry);
401 	KASSERT(obj == NULL);
402 
403 	sodopendfree();
404 	if (!vm_map_starved_p(kernel_map)) {
405 		return CALLBACK_CHAIN_ABORT;
406 	}
407 	return CALLBACK_CHAIN_CONTINUE;
408 }
409 
410 struct mbuf *
411 getsombuf(struct socket *so, int type)
412 {
413 	struct mbuf *m;
414 
415 	m = m_get(M_WAIT, type);
416 	MCLAIM(m, so->so_mowner);
417 	return m;
418 }
419 
420 struct mbuf *
421 m_intopt(struct socket *so, int val)
422 {
423 	struct mbuf *m;
424 
425 	m = getsombuf(so, MT_SOOPTS);
426 	m->m_len = sizeof(int);
427 	*mtod(m, int *) = val;
428 	return m;
429 }
430 
431 void
432 soinit(void)
433 {
434 
435 	mutex_init(&so_pendfree_lock, MUTEX_DEFAULT, IPL_VM);
436 	softnet_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
437 	cv_init(&socurkva_cv, "sokva");
438 
439 	/* Set the initial adjusted socket buffer size. */
440 	if (sb_max_set(sb_max))
441 		panic("bad initial sb_max value: %lu", sb_max);
442 
443 	callback_register(&vm_map_to_kernel(kernel_map)->vmk_reclaim_callback,
444 	    &sokva_reclaimerentry, NULL, sokva_reclaim_callback);
445 }
446 
447 /*
448  * Socket operation routines.
449  * These routines are called by the routines in
450  * sys_socket.c or from a system process, and
451  * implement the semantics of socket operations by
452  * switching out to the protocol specific routines.
453  */
454 /*ARGSUSED*/
455 int
456 socreate(int dom, struct socket **aso, int type, int proto, struct lwp *l,
457 	 struct socket *lockso)
458 {
459 	const struct protosw	*prp;
460 	struct socket	*so;
461 	uid_t		uid;
462 	int		error;
463 	kmutex_t	*lock;
464 
465 	error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_SOCKET,
466 	    KAUTH_REQ_NETWORK_SOCKET_OPEN, KAUTH_ARG(dom), KAUTH_ARG(type),
467 	    KAUTH_ARG(proto));
468 	if (error != 0)
469 		return error;
470 
471 	if (proto)
472 		prp = pffindproto(dom, proto, type);
473 	else
474 		prp = pffindtype(dom, type);
475 	if (prp == NULL) {
476 		/* no support for domain */
477 		if (pffinddomain(dom) == 0)
478 			return EAFNOSUPPORT;
479 		/* no support for socket type */
480 		if (proto == 0 && type != 0)
481 			return EPROTOTYPE;
482 		return EPROTONOSUPPORT;
483 	}
484 	if (prp->pr_usrreq == NULL)
485 		return EPROTONOSUPPORT;
486 	if (prp->pr_type != type)
487 		return EPROTOTYPE;
488 
489 	so = soget(true);
490 	so->so_type = type;
491 	so->so_proto = prp;
492 	so->so_send = sosend;
493 	so->so_receive = soreceive;
494 #ifdef MBUFTRACE
495 	so->so_rcv.sb_mowner = &prp->pr_domain->dom_mowner;
496 	so->so_snd.sb_mowner = &prp->pr_domain->dom_mowner;
497 	so->so_mowner = &prp->pr_domain->dom_mowner;
498 #endif
499 	uid = kauth_cred_geteuid(l->l_cred);
500 	so->so_uidinfo = uid_find(uid);
501 	if (lockso != NULL) {
502 		/* Caller wants us to share a lock. */
503 		lock = lockso->so_lock;
504 		so->so_lock = lock;
505 		mutex_obj_hold(lock);
506 		mutex_enter(lock);
507 	} else {
508 		/* Lock assigned and taken during PRU_ATTACH. */
509 	}
510 	error = (*prp->pr_usrreq)(so, PRU_ATTACH, NULL,
511 	    (struct mbuf *)(long)proto, NULL, l);
512 	KASSERT(solocked(so));
513 	if (error != 0) {
514 		so->so_state |= SS_NOFDREF;
515 		sofree(so);
516 		return error;
517 	}
518 	sounlock(so);
519 	*aso = so;
520 	return 0;
521 }
522 
523 /* On success, write file descriptor to fdout and return zero.  On
524  * failure, return non-zero; *fdout will be undefined.
525  */
526 int
527 fsocreate(int domain, struct socket **sop, int type, int protocol,
528     struct lwp *l, int *fdout)
529 {
530 	struct socket	*so;
531 	struct file	*fp;
532 	int		fd, error;
533 
534 	if ((error = fd_allocfile(&fp, &fd)) != 0)
535 		return (error);
536 	fp->f_flag = FREAD|FWRITE;
537 	fp->f_type = DTYPE_SOCKET;
538 	fp->f_ops = &socketops;
539 	error = socreate(domain, &so, type, protocol, l, NULL);
540 	if (error != 0) {
541 		fd_abort(curproc, fp, fd);
542 	} else {
543 		if (sop != NULL)
544 			*sop = so;
545 		fp->f_data = so;
546 		fd_affix(curproc, fp, fd);
547 		*fdout = fd;
548 	}
549 	return error;
550 }
551 
552 int
553 sobind(struct socket *so, struct mbuf *nam, struct lwp *l)
554 {
555 	int	error;
556 
557 	solock(so);
558 	error = (*so->so_proto->pr_usrreq)(so, PRU_BIND, NULL, nam, NULL, l);
559 	sounlock(so);
560 	return error;
561 }
562 
563 int
564 solisten(struct socket *so, int backlog, struct lwp *l)
565 {
566 	int	error;
567 
568 	solock(so);
569 	if ((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING |
570 	    SS_ISDISCONNECTING)) != 0)
571 		return (EOPNOTSUPP);
572 	error = (*so->so_proto->pr_usrreq)(so, PRU_LISTEN, NULL,
573 	    NULL, NULL, l);
574 	if (error != 0) {
575 		sounlock(so);
576 		return error;
577 	}
578 	if (TAILQ_EMPTY(&so->so_q))
579 		so->so_options |= SO_ACCEPTCONN;
580 	if (backlog < 0)
581 		backlog = 0;
582 	so->so_qlimit = min(backlog, somaxconn);
583 	sounlock(so);
584 	return 0;
585 }
586 
587 void
588 sofree(struct socket *so)
589 {
590 	u_int refs;
591 
592 	KASSERT(solocked(so));
593 
594 	if (so->so_pcb || (so->so_state & SS_NOFDREF) == 0) {
595 		sounlock(so);
596 		return;
597 	}
598 	if (so->so_head) {
599 		/*
600 		 * We must not decommission a socket that's on the accept(2)
601 		 * queue.  If we do, then accept(2) may hang after select(2)
602 		 * indicated that the listening socket was ready.
603 		 */
604 		if (!soqremque(so, 0)) {
605 			sounlock(so);
606 			return;
607 		}
608 	}
609 	if (so->so_rcv.sb_hiwat)
610 		(void)chgsbsize(so->so_uidinfo, &so->so_rcv.sb_hiwat, 0,
611 		    RLIM_INFINITY);
612 	if (so->so_snd.sb_hiwat)
613 		(void)chgsbsize(so->so_uidinfo, &so->so_snd.sb_hiwat, 0,
614 		    RLIM_INFINITY);
615 	sbrelease(&so->so_snd, so);
616 	KASSERT(!cv_has_waiters(&so->so_cv));
617 	KASSERT(!cv_has_waiters(&so->so_rcv.sb_cv));
618 	KASSERT(!cv_has_waiters(&so->so_snd.sb_cv));
619 	sorflush(so);
620 	refs = so->so_aborting;	/* XXX */
621 	sounlock(so);
622 	if (refs == 0)		/* XXX */
623 		soput(so);
624 }
625 
626 /*
627  * Close a socket on last file table reference removal.
628  * Initiate disconnect if connected.
629  * Free socket when disconnect complete.
630  */
631 int
632 soclose(struct socket *so)
633 {
634 	struct socket	*so2;
635 	int		error;
636 	int		error2;
637 
638 	error = 0;
639 	solock(so);
640 	if (so->so_options & SO_ACCEPTCONN) {
641 		do {
642 			if ((so2 = TAILQ_FIRST(&so->so_q0)) != 0) {
643 				KASSERT(solocked2(so, so2));
644 				(void) soqremque(so2, 0);
645 				/* soabort drops the lock. */
646 				(void) soabort(so2);
647 				solock(so);
648 				continue;
649 			}
650 			if ((so2 = TAILQ_FIRST(&so->so_q)) != 0) {
651 				KASSERT(solocked2(so, so2));
652 				(void) soqremque(so2, 1);
653 				/* soabort drops the lock. */
654 				(void) soabort(so2);
655 				solock(so);
656 				continue;
657 			}
658 		} while (0);
659 	}
660 	if (so->so_pcb == 0)
661 		goto discard;
662 	if (so->so_state & SS_ISCONNECTED) {
663 		if ((so->so_state & SS_ISDISCONNECTING) == 0) {
664 			error = sodisconnect(so);
665 			if (error)
666 				goto drop;
667 		}
668 		if (so->so_options & SO_LINGER) {
669 			if ((so->so_state & SS_ISDISCONNECTING) && so->so_nbio)
670 				goto drop;
671 			while (so->so_state & SS_ISCONNECTED) {
672 				error = sowait(so, so->so_linger * hz);
673 				if (error)
674 					break;
675 			}
676 		}
677 	}
678  drop:
679 	if (so->so_pcb) {
680 		error2 = (*so->so_proto->pr_usrreq)(so, PRU_DETACH,
681 		    NULL, NULL, NULL, NULL);
682 		if (error == 0)
683 			error = error2;
684 	}
685  discard:
686 	if (so->so_state & SS_NOFDREF)
687 		panic("soclose: NOFDREF");
688 	so->so_state |= SS_NOFDREF;
689 	sofree(so);
690 	return (error);
691 }
692 
693 /*
694  * Must be called with the socket locked..  Will return with it unlocked.
695  */
696 int
697 soabort(struct socket *so)
698 {
699 	u_int refs;
700 	int error;
701 
702 	KASSERT(solocked(so));
703 	KASSERT(so->so_head == NULL);
704 
705 	so->so_aborting++;		/* XXX */
706 	error = (*so->so_proto->pr_usrreq)(so, PRU_ABORT, NULL,
707 	    NULL, NULL, NULL);
708 	refs = --so->so_aborting;	/* XXX */
709 	if (error) {
710 		sofree(so);
711 	} else {
712 		sounlock(so);
713 		if (refs == 0)
714 			sofree(so);
715 	}
716 	return error;
717 }
718 
719 int
720 soaccept(struct socket *so, struct mbuf *nam)
721 {
722 	int	error;
723 
724 	KASSERT(solocked(so));
725 
726 	error = 0;
727 	if ((so->so_state & SS_NOFDREF) == 0)
728 		panic("soaccept: !NOFDREF");
729 	so->so_state &= ~SS_NOFDREF;
730 	if ((so->so_state & SS_ISDISCONNECTED) == 0 ||
731 	    (so->so_proto->pr_flags & PR_ABRTACPTDIS) == 0)
732 		error = (*so->so_proto->pr_usrreq)(so, PRU_ACCEPT,
733 		    NULL, nam, NULL, NULL);
734 	else
735 		error = ECONNABORTED;
736 
737 	return (error);
738 }
739 
740 int
741 soconnect(struct socket *so, struct mbuf *nam, struct lwp *l)
742 {
743 	int		error;
744 
745 	KASSERT(solocked(so));
746 
747 	if (so->so_options & SO_ACCEPTCONN)
748 		return (EOPNOTSUPP);
749 	/*
750 	 * If protocol is connection-based, can only connect once.
751 	 * Otherwise, if connected, try to disconnect first.
752 	 * This allows user to disconnect by connecting to, e.g.,
753 	 * a null address.
754 	 */
755 	if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) &&
756 	    ((so->so_proto->pr_flags & PR_CONNREQUIRED) ||
757 	    (error = sodisconnect(so))))
758 		error = EISCONN;
759 	else
760 		error = (*so->so_proto->pr_usrreq)(so, PRU_CONNECT,
761 		    NULL, nam, NULL, l);
762 	return (error);
763 }
764 
765 int
766 soconnect2(struct socket *so1, struct socket *so2)
767 {
768 	int	error;
769 
770 	KASSERT(solocked2(so1, so2));
771 
772 	error = (*so1->so_proto->pr_usrreq)(so1, PRU_CONNECT2,
773 	    NULL, (struct mbuf *)so2, NULL, NULL);
774 	return (error);
775 }
776 
777 int
778 sodisconnect(struct socket *so)
779 {
780 	int	error;
781 
782 	KASSERT(solocked(so));
783 
784 	if ((so->so_state & SS_ISCONNECTED) == 0) {
785 		error = ENOTCONN;
786 	} else if (so->so_state & SS_ISDISCONNECTING) {
787 		error = EALREADY;
788 	} else {
789 		error = (*so->so_proto->pr_usrreq)(so, PRU_DISCONNECT,
790 		    NULL, NULL, NULL, NULL);
791 	}
792 	sodopendfree();
793 	return (error);
794 }
795 
796 #define	SBLOCKWAIT(f)	(((f) & MSG_DONTWAIT) ? M_NOWAIT : M_WAITOK)
797 /*
798  * Send on a socket.
799  * If send must go all at once and message is larger than
800  * send buffering, then hard error.
801  * Lock against other senders.
802  * If must go all at once and not enough room now, then
803  * inform user that this would block and do nothing.
804  * Otherwise, if nonblocking, send as much as possible.
805  * The data to be sent is described by "uio" if nonzero,
806  * otherwise by the mbuf chain "top" (which must be null
807  * if uio is not).  Data provided in mbuf chain must be small
808  * enough to send all at once.
809  *
810  * Returns nonzero on error, timeout or signal; callers
811  * must check for short counts if EINTR/ERESTART are returned.
812  * Data and control buffers are freed on return.
813  */
814 int
815 sosend(struct socket *so, struct mbuf *addr, struct uio *uio, struct mbuf *top,
816 	struct mbuf *control, int flags, struct lwp *l)
817 {
818 	struct mbuf	**mp, *m;
819 	struct proc	*p;
820 	long		space, len, resid, clen, mlen;
821 	int		error, s, dontroute, atomic;
822 
823 	p = l->l_proc;
824 	sodopendfree();
825 	clen = 0;
826 
827 	/*
828 	 * solock() provides atomicity of access.  splsoftnet() prevents
829 	 * protocol processing soft interrupts from interrupting us and
830 	 * blocking (expensive).
831 	 */
832 	s = splsoftnet();
833 	solock(so);
834 	atomic = sosendallatonce(so) || top;
835 	if (uio)
836 		resid = uio->uio_resid;
837 	else
838 		resid = top->m_pkthdr.len;
839 	/*
840 	 * In theory resid should be unsigned.
841 	 * However, space must be signed, as it might be less than 0
842 	 * if we over-committed, and we must use a signed comparison
843 	 * of space and resid.  On the other hand, a negative resid
844 	 * causes us to loop sending 0-length segments to the protocol.
845 	 */
846 	if (resid < 0) {
847 		error = EINVAL;
848 		goto out;
849 	}
850 	dontroute =
851 	    (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 &&
852 	    (so->so_proto->pr_flags & PR_ATOMIC);
853 	if (l)
854 		l->l_ru.ru_msgsnd++;
855 	if (control)
856 		clen = control->m_len;
857  restart:
858 	if ((error = sblock(&so->so_snd, SBLOCKWAIT(flags))) != 0)
859 		goto out;
860 	do {
861 		if (so->so_state & SS_CANTSENDMORE) {
862 			error = EPIPE;
863 			goto release;
864 		}
865 		if (so->so_error) {
866 			error = so->so_error;
867 			so->so_error = 0;
868 			goto release;
869 		}
870 		if ((so->so_state & SS_ISCONNECTED) == 0) {
871 			if (so->so_proto->pr_flags & PR_CONNREQUIRED) {
872 				if ((so->so_state & SS_ISCONFIRMING) == 0 &&
873 				    !(resid == 0 && clen != 0)) {
874 					error = ENOTCONN;
875 					goto release;
876 				}
877 			} else if (addr == 0) {
878 				error = EDESTADDRREQ;
879 				goto release;
880 			}
881 		}
882 		space = sbspace(&so->so_snd);
883 		if (flags & MSG_OOB)
884 			space += 1024;
885 		if ((atomic && resid > so->so_snd.sb_hiwat) ||
886 		    clen > so->so_snd.sb_hiwat) {
887 			error = EMSGSIZE;
888 			goto release;
889 		}
890 		if (space < resid + clen &&
891 		    (atomic || space < so->so_snd.sb_lowat || space < clen)) {
892 			if (so->so_nbio) {
893 				error = EWOULDBLOCK;
894 				goto release;
895 			}
896 			sbunlock(&so->so_snd);
897 			error = sbwait(&so->so_snd);
898 			if (error)
899 				goto out;
900 			goto restart;
901 		}
902 		mp = &top;
903 		space -= clen;
904 		do {
905 			if (uio == NULL) {
906 				/*
907 				 * Data is prepackaged in "top".
908 				 */
909 				resid = 0;
910 				if (flags & MSG_EOR)
911 					top->m_flags |= M_EOR;
912 			} else do {
913 				sounlock(so);
914 				splx(s);
915 				if (top == NULL) {
916 					m = m_gethdr(M_WAIT, MT_DATA);
917 					mlen = MHLEN;
918 					m->m_pkthdr.len = 0;
919 					m->m_pkthdr.rcvif = NULL;
920 				} else {
921 					m = m_get(M_WAIT, MT_DATA);
922 					mlen = MLEN;
923 				}
924 				MCLAIM(m, so->so_snd.sb_mowner);
925 				if (sock_loan_thresh >= 0 &&
926 				    uio->uio_iov->iov_len >= sock_loan_thresh &&
927 				    space >= sock_loan_thresh &&
928 				    (len = sosend_loan(so, uio, m,
929 						       space)) != 0) {
930 					SOSEND_COUNTER_INCR(&sosend_loan_big);
931 					space -= len;
932 					goto have_data;
933 				}
934 				if (resid >= MINCLSIZE && space >= MCLBYTES) {
935 					SOSEND_COUNTER_INCR(&sosend_copy_big);
936 					m_clget(m, M_WAIT);
937 					if ((m->m_flags & M_EXT) == 0)
938 						goto nopages;
939 					mlen = MCLBYTES;
940 					if (atomic && top == 0) {
941 						len = lmin(MCLBYTES - max_hdr,
942 						    resid);
943 						m->m_data += max_hdr;
944 					} else
945 						len = lmin(MCLBYTES, resid);
946 					space -= len;
947 				} else {
948  nopages:
949 					SOSEND_COUNTER_INCR(&sosend_copy_small);
950 					len = lmin(lmin(mlen, resid), space);
951 					space -= len;
952 					/*
953 					 * For datagram protocols, leave room
954 					 * for protocol headers in first mbuf.
955 					 */
956 					if (atomic && top == 0 && len < mlen)
957 						MH_ALIGN(m, len);
958 				}
959 				error = uiomove(mtod(m, void *), (int)len, uio);
960  have_data:
961 				resid = uio->uio_resid;
962 				m->m_len = len;
963 				*mp = m;
964 				top->m_pkthdr.len += len;
965 				s = splsoftnet();
966 				solock(so);
967 				if (error != 0)
968 					goto release;
969 				mp = &m->m_next;
970 				if (resid <= 0) {
971 					if (flags & MSG_EOR)
972 						top->m_flags |= M_EOR;
973 					break;
974 				}
975 			} while (space > 0 && atomic);
976 
977 			if (so->so_state & SS_CANTSENDMORE) {
978 				error = EPIPE;
979 				goto release;
980 			}
981 			if (dontroute)
982 				so->so_options |= SO_DONTROUTE;
983 			if (resid > 0)
984 				so->so_state |= SS_MORETOCOME;
985 			error = (*so->so_proto->pr_usrreq)(so,
986 			    (flags & MSG_OOB) ? PRU_SENDOOB : PRU_SEND,
987 			    top, addr, control, curlwp);
988 			if (dontroute)
989 				so->so_options &= ~SO_DONTROUTE;
990 			if (resid > 0)
991 				so->so_state &= ~SS_MORETOCOME;
992 			clen = 0;
993 			control = NULL;
994 			top = NULL;
995 			mp = &top;
996 			if (error != 0)
997 				goto release;
998 		} while (resid && space > 0);
999 	} while (resid);
1000 
1001  release:
1002 	sbunlock(&so->so_snd);
1003  out:
1004 	sounlock(so);
1005 	splx(s);
1006 	if (top)
1007 		m_freem(top);
1008 	if (control)
1009 		m_freem(control);
1010 	return (error);
1011 }
1012 
1013 /*
1014  * Following replacement or removal of the first mbuf on the first
1015  * mbuf chain of a socket buffer, push necessary state changes back
1016  * into the socket buffer so that other consumers see the values
1017  * consistently.  'nextrecord' is the callers locally stored value of
1018  * the original value of sb->sb_mb->m_nextpkt which must be restored
1019  * when the lead mbuf changes.  NOTE: 'nextrecord' may be NULL.
1020  */
1021 static void
1022 sbsync(struct sockbuf *sb, struct mbuf *nextrecord)
1023 {
1024 
1025 	KASSERT(solocked(sb->sb_so));
1026 
1027 	/*
1028 	 * First, update for the new value of nextrecord.  If necessary,
1029 	 * make it the first record.
1030 	 */
1031 	if (sb->sb_mb != NULL)
1032 		sb->sb_mb->m_nextpkt = nextrecord;
1033 	else
1034 		sb->sb_mb = nextrecord;
1035 
1036         /*
1037          * Now update any dependent socket buffer fields to reflect
1038          * the new state.  This is an inline of SB_EMPTY_FIXUP, with
1039          * the addition of a second clause that takes care of the
1040          * case where sb_mb has been updated, but remains the last
1041          * record.
1042          */
1043         if (sb->sb_mb == NULL) {
1044                 sb->sb_mbtail = NULL;
1045                 sb->sb_lastrecord = NULL;
1046         } else if (sb->sb_mb->m_nextpkt == NULL)
1047                 sb->sb_lastrecord = sb->sb_mb;
1048 }
1049 
1050 /*
1051  * Implement receive operations on a socket.
1052  * We depend on the way that records are added to the sockbuf
1053  * by sbappend*.  In particular, each record (mbufs linked through m_next)
1054  * must begin with an address if the protocol so specifies,
1055  * followed by an optional mbuf or mbufs containing ancillary data,
1056  * and then zero or more mbufs of data.
1057  * In order to avoid blocking network interrupts for the entire time here,
1058  * we splx() while doing the actual copy to user space.
1059  * Although the sockbuf is locked, new data may still be appended,
1060  * and thus we must maintain consistency of the sockbuf during that time.
1061  *
1062  * The caller may receive the data as a single mbuf chain by supplying
1063  * an mbuf **mp0 for use in returning the chain.  The uio is then used
1064  * only for the count in uio_resid.
1065  */
1066 int
1067 soreceive(struct socket *so, struct mbuf **paddr, struct uio *uio,
1068 	struct mbuf **mp0, struct mbuf **controlp, int *flagsp)
1069 {
1070 	struct lwp *l = curlwp;
1071 	struct mbuf	*m, **mp, *mt;
1072 	int atomic, flags, len, error, s, offset, moff, type, orig_resid;
1073 	const struct protosw	*pr;
1074 	struct mbuf	*nextrecord;
1075 	int		mbuf_removed = 0;
1076 	const struct domain *dom;
1077 
1078 	pr = so->so_proto;
1079 	atomic = pr->pr_flags & PR_ATOMIC;
1080 	dom = pr->pr_domain;
1081 	mp = mp0;
1082 	type = 0;
1083 	orig_resid = uio->uio_resid;
1084 
1085 	if (paddr != NULL)
1086 		*paddr = NULL;
1087 	if (controlp != NULL)
1088 		*controlp = NULL;
1089 	if (flagsp != NULL)
1090 		flags = *flagsp &~ MSG_EOR;
1091 	else
1092 		flags = 0;
1093 
1094 	if ((flags & MSG_DONTWAIT) == 0)
1095 		sodopendfree();
1096 
1097 	if (flags & MSG_OOB) {
1098 		m = m_get(M_WAIT, MT_DATA);
1099 		solock(so);
1100 		error = (*pr->pr_usrreq)(so, PRU_RCVOOB, m,
1101 		    (struct mbuf *)(long)(flags & MSG_PEEK), NULL, l);
1102 		sounlock(so);
1103 		if (error)
1104 			goto bad;
1105 		do {
1106 			error = uiomove(mtod(m, void *),
1107 			    (int) min(uio->uio_resid, m->m_len), uio);
1108 			m = m_free(m);
1109 		} while (uio->uio_resid > 0 && error == 0 && m);
1110  bad:
1111 		if (m != NULL)
1112 			m_freem(m);
1113 		return error;
1114 	}
1115 	if (mp != NULL)
1116 		*mp = NULL;
1117 
1118 	/*
1119 	 * solock() provides atomicity of access.  splsoftnet() prevents
1120 	 * protocol processing soft interrupts from interrupting us and
1121 	 * blocking (expensive).
1122 	 */
1123 	s = splsoftnet();
1124 	solock(so);
1125 	if (so->so_state & SS_ISCONFIRMING && uio->uio_resid)
1126 		(*pr->pr_usrreq)(so, PRU_RCVD, NULL, NULL, NULL, l);
1127 
1128  restart:
1129 	if ((error = sblock(&so->so_rcv, SBLOCKWAIT(flags))) != 0) {
1130 		sounlock(so);
1131 		splx(s);
1132 		return error;
1133 	}
1134 
1135 	m = so->so_rcv.sb_mb;
1136 	/*
1137 	 * If we have less data than requested, block awaiting more
1138 	 * (subject to any timeout) if:
1139 	 *   1. the current count is less than the low water mark,
1140 	 *   2. MSG_WAITALL is set, and it is possible to do the entire
1141 	 *	receive operation at once if we block (resid <= hiwat), or
1142 	 *   3. MSG_DONTWAIT is not set.
1143 	 * If MSG_WAITALL is set but resid is larger than the receive buffer,
1144 	 * we have to do the receive in sections, and thus risk returning
1145 	 * a short count if a timeout or signal occurs after we start.
1146 	 */
1147 	if (m == NULL ||
1148 	    ((flags & MSG_DONTWAIT) == 0 &&
1149 	     so->so_rcv.sb_cc < uio->uio_resid &&
1150 	     (so->so_rcv.sb_cc < so->so_rcv.sb_lowat ||
1151 	      ((flags & MSG_WAITALL) &&
1152 	       uio->uio_resid <= so->so_rcv.sb_hiwat)) &&
1153 	     m->m_nextpkt == NULL && !atomic)) {
1154 #ifdef DIAGNOSTIC
1155 		if (m == NULL && so->so_rcv.sb_cc)
1156 			panic("receive 1");
1157 #endif
1158 		if (so->so_error) {
1159 			if (m != NULL)
1160 				goto dontblock;
1161 			error = so->so_error;
1162 			if ((flags & MSG_PEEK) == 0)
1163 				so->so_error = 0;
1164 			goto release;
1165 		}
1166 		if (so->so_state & SS_CANTRCVMORE) {
1167 			if (m != NULL)
1168 				goto dontblock;
1169 			else
1170 				goto release;
1171 		}
1172 		for (; m != NULL; m = m->m_next)
1173 			if (m->m_type == MT_OOBDATA  || (m->m_flags & M_EOR)) {
1174 				m = so->so_rcv.sb_mb;
1175 				goto dontblock;
1176 			}
1177 		if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 &&
1178 		    (so->so_proto->pr_flags & PR_CONNREQUIRED)) {
1179 			error = ENOTCONN;
1180 			goto release;
1181 		}
1182 		if (uio->uio_resid == 0)
1183 			goto release;
1184 		if (so->so_nbio || (flags & MSG_DONTWAIT)) {
1185 			error = EWOULDBLOCK;
1186 			goto release;
1187 		}
1188 		SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 1");
1189 		SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 1");
1190 		sbunlock(&so->so_rcv);
1191 		error = sbwait(&so->so_rcv);
1192 		if (error != 0) {
1193 			sounlock(so);
1194 			splx(s);
1195 			return error;
1196 		}
1197 		goto restart;
1198 	}
1199  dontblock:
1200 	/*
1201 	 * On entry here, m points to the first record of the socket buffer.
1202 	 * From this point onward, we maintain 'nextrecord' as a cache of the
1203 	 * pointer to the next record in the socket buffer.  We must keep the
1204 	 * various socket buffer pointers and local stack versions of the
1205 	 * pointers in sync, pushing out modifications before dropping the
1206 	 * socket lock, and re-reading them when picking it up.
1207 	 *
1208 	 * Otherwise, we will race with the network stack appending new data
1209 	 * or records onto the socket buffer by using inconsistent/stale
1210 	 * versions of the field, possibly resulting in socket buffer
1211 	 * corruption.
1212 	 *
1213 	 * By holding the high-level sblock(), we prevent simultaneous
1214 	 * readers from pulling off the front of the socket buffer.
1215 	 */
1216 	if (l != NULL)
1217 		l->l_ru.ru_msgrcv++;
1218 	KASSERT(m == so->so_rcv.sb_mb);
1219 	SBLASTRECORDCHK(&so->so_rcv, "soreceive 1");
1220 	SBLASTMBUFCHK(&so->so_rcv, "soreceive 1");
1221 	nextrecord = m->m_nextpkt;
1222 	if (pr->pr_flags & PR_ADDR) {
1223 #ifdef DIAGNOSTIC
1224 		if (m->m_type != MT_SONAME)
1225 			panic("receive 1a");
1226 #endif
1227 		orig_resid = 0;
1228 		if (flags & MSG_PEEK) {
1229 			if (paddr)
1230 				*paddr = m_copy(m, 0, m->m_len);
1231 			m = m->m_next;
1232 		} else {
1233 			sbfree(&so->so_rcv, m);
1234 			mbuf_removed = 1;
1235 			if (paddr != NULL) {
1236 				*paddr = m;
1237 				so->so_rcv.sb_mb = m->m_next;
1238 				m->m_next = NULL;
1239 				m = so->so_rcv.sb_mb;
1240 			} else {
1241 				MFREE(m, so->so_rcv.sb_mb);
1242 				m = so->so_rcv.sb_mb;
1243 			}
1244 			sbsync(&so->so_rcv, nextrecord);
1245 		}
1246 	}
1247 
1248 	/*
1249 	 * Process one or more MT_CONTROL mbufs present before any data mbufs
1250 	 * in the first mbuf chain on the socket buffer.  If MSG_PEEK, we
1251 	 * just copy the data; if !MSG_PEEK, we call into the protocol to
1252 	 * perform externalization (or freeing if controlp == NULL).
1253 	 */
1254 	if (__predict_false(m != NULL && m->m_type == MT_CONTROL)) {
1255 		struct mbuf *cm = NULL, *cmn;
1256 		struct mbuf **cme = &cm;
1257 
1258 		do {
1259 			if (flags & MSG_PEEK) {
1260 				if (controlp != NULL) {
1261 					*controlp = m_copy(m, 0, m->m_len);
1262 					controlp = &(*controlp)->m_next;
1263 				}
1264 				m = m->m_next;
1265 			} else {
1266 				sbfree(&so->so_rcv, m);
1267 				so->so_rcv.sb_mb = m->m_next;
1268 				m->m_next = NULL;
1269 				*cme = m;
1270 				cme = &(*cme)->m_next;
1271 				m = so->so_rcv.sb_mb;
1272 			}
1273 		} while (m != NULL && m->m_type == MT_CONTROL);
1274 		if ((flags & MSG_PEEK) == 0)
1275 			sbsync(&so->so_rcv, nextrecord);
1276 		for (; cm != NULL; cm = cmn) {
1277 			cmn = cm->m_next;
1278 			cm->m_next = NULL;
1279 			type = mtod(cm, struct cmsghdr *)->cmsg_type;
1280 			if (controlp != NULL) {
1281 				if (dom->dom_externalize != NULL &&
1282 				    type == SCM_RIGHTS) {
1283 					sounlock(so);
1284 					splx(s);
1285 					error = (*dom->dom_externalize)(cm, l);
1286 					s = splsoftnet();
1287 					solock(so);
1288 				}
1289 				*controlp = cm;
1290 				while (*controlp != NULL)
1291 					controlp = &(*controlp)->m_next;
1292 			} else {
1293 				/*
1294 				 * Dispose of any SCM_RIGHTS message that went
1295 				 * through the read path rather than recv.
1296 				 */
1297 				if (dom->dom_dispose != NULL &&
1298 				    type == SCM_RIGHTS) {
1299 				    	sounlock(so);
1300 					(*dom->dom_dispose)(cm);
1301 					solock(so);
1302 				}
1303 				m_freem(cm);
1304 			}
1305 		}
1306 		if (m != NULL)
1307 			nextrecord = so->so_rcv.sb_mb->m_nextpkt;
1308 		else
1309 			nextrecord = so->so_rcv.sb_mb;
1310 		orig_resid = 0;
1311 	}
1312 
1313 	/* If m is non-NULL, we have some data to read. */
1314 	if (__predict_true(m != NULL)) {
1315 		type = m->m_type;
1316 		if (type == MT_OOBDATA)
1317 			flags |= MSG_OOB;
1318 	}
1319 	SBLASTRECORDCHK(&so->so_rcv, "soreceive 2");
1320 	SBLASTMBUFCHK(&so->so_rcv, "soreceive 2");
1321 
1322 	moff = 0;
1323 	offset = 0;
1324 	while (m != NULL && uio->uio_resid > 0 && error == 0) {
1325 		if (m->m_type == MT_OOBDATA) {
1326 			if (type != MT_OOBDATA)
1327 				break;
1328 		} else if (type == MT_OOBDATA)
1329 			break;
1330 #ifdef DIAGNOSTIC
1331 		else if (m->m_type != MT_DATA && m->m_type != MT_HEADER)
1332 			panic("receive 3");
1333 #endif
1334 		so->so_state &= ~SS_RCVATMARK;
1335 		len = uio->uio_resid;
1336 		if (so->so_oobmark && len > so->so_oobmark - offset)
1337 			len = so->so_oobmark - offset;
1338 		if (len > m->m_len - moff)
1339 			len = m->m_len - moff;
1340 		/*
1341 		 * If mp is set, just pass back the mbufs.
1342 		 * Otherwise copy them out via the uio, then free.
1343 		 * Sockbuf must be consistent here (points to current mbuf,
1344 		 * it points to next record) when we drop priority;
1345 		 * we must note any additions to the sockbuf when we
1346 		 * block interrupts again.
1347 		 */
1348 		if (mp == NULL) {
1349 			SBLASTRECORDCHK(&so->so_rcv, "soreceive uiomove");
1350 			SBLASTMBUFCHK(&so->so_rcv, "soreceive uiomove");
1351 			sounlock(so);
1352 			splx(s);
1353 			error = uiomove(mtod(m, char *) + moff, (int)len, uio);
1354 			s = splsoftnet();
1355 			solock(so);
1356 			if (error != 0) {
1357 				/*
1358 				 * If any part of the record has been removed
1359 				 * (such as the MT_SONAME mbuf, which will
1360 				 * happen when PR_ADDR, and thus also
1361 				 * PR_ATOMIC, is set), then drop the entire
1362 				 * record to maintain the atomicity of the
1363 				 * receive operation.
1364 				 *
1365 				 * This avoids a later panic("receive 1a")
1366 				 * when compiled with DIAGNOSTIC.
1367 				 */
1368 				if (m && mbuf_removed && atomic)
1369 					(void) sbdroprecord(&so->so_rcv);
1370 
1371 				goto release;
1372 			}
1373 		} else
1374 			uio->uio_resid -= len;
1375 		if (len == m->m_len - moff) {
1376 			if (m->m_flags & M_EOR)
1377 				flags |= MSG_EOR;
1378 			if (flags & MSG_PEEK) {
1379 				m = m->m_next;
1380 				moff = 0;
1381 			} else {
1382 				nextrecord = m->m_nextpkt;
1383 				sbfree(&so->so_rcv, m);
1384 				if (mp) {
1385 					*mp = m;
1386 					mp = &m->m_next;
1387 					so->so_rcv.sb_mb = m = m->m_next;
1388 					*mp = NULL;
1389 				} else {
1390 					MFREE(m, so->so_rcv.sb_mb);
1391 					m = so->so_rcv.sb_mb;
1392 				}
1393 				/*
1394 				 * If m != NULL, we also know that
1395 				 * so->so_rcv.sb_mb != NULL.
1396 				 */
1397 				KASSERT(so->so_rcv.sb_mb == m);
1398 				if (m) {
1399 					m->m_nextpkt = nextrecord;
1400 					if (nextrecord == NULL)
1401 						so->so_rcv.sb_lastrecord = m;
1402 				} else {
1403 					so->so_rcv.sb_mb = nextrecord;
1404 					SB_EMPTY_FIXUP(&so->so_rcv);
1405 				}
1406 				SBLASTRECORDCHK(&so->so_rcv, "soreceive 3");
1407 				SBLASTMBUFCHK(&so->so_rcv, "soreceive 3");
1408 			}
1409 		} else if (flags & MSG_PEEK)
1410 			moff += len;
1411 		else {
1412 			if (mp != NULL) {
1413 				mt = m_copym(m, 0, len, M_NOWAIT);
1414 				if (__predict_false(mt == NULL)) {
1415 					sounlock(so);
1416 					mt = m_copym(m, 0, len, M_WAIT);
1417 					solock(so);
1418 				}
1419 				*mp = mt;
1420 			}
1421 			m->m_data += len;
1422 			m->m_len -= len;
1423 			so->so_rcv.sb_cc -= len;
1424 		}
1425 		if (so->so_oobmark) {
1426 			if ((flags & MSG_PEEK) == 0) {
1427 				so->so_oobmark -= len;
1428 				if (so->so_oobmark == 0) {
1429 					so->so_state |= SS_RCVATMARK;
1430 					break;
1431 				}
1432 			} else {
1433 				offset += len;
1434 				if (offset == so->so_oobmark)
1435 					break;
1436 			}
1437 		}
1438 		if (flags & MSG_EOR)
1439 			break;
1440 		/*
1441 		 * If the MSG_WAITALL flag is set (for non-atomic socket),
1442 		 * we must not quit until "uio->uio_resid == 0" or an error
1443 		 * termination.  If a signal/timeout occurs, return
1444 		 * with a short count but without error.
1445 		 * Keep sockbuf locked against other readers.
1446 		 */
1447 		while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 &&
1448 		    !sosendallatonce(so) && !nextrecord) {
1449 			if (so->so_error || so->so_state & SS_CANTRCVMORE)
1450 				break;
1451 			/*
1452 			 * If we are peeking and the socket receive buffer is
1453 			 * full, stop since we can't get more data to peek at.
1454 			 */
1455 			if ((flags & MSG_PEEK) && sbspace(&so->so_rcv) <= 0)
1456 				break;
1457 			/*
1458 			 * If we've drained the socket buffer, tell the
1459 			 * protocol in case it needs to do something to
1460 			 * get it filled again.
1461 			 */
1462 			if ((pr->pr_flags & PR_WANTRCVD) && so->so_pcb)
1463 				(*pr->pr_usrreq)(so, PRU_RCVD,
1464 				    NULL, (struct mbuf *)(long)flags, NULL, l);
1465 			SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 2");
1466 			SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 2");
1467 			error = sbwait(&so->so_rcv);
1468 			if (error != 0) {
1469 				sbunlock(&so->so_rcv);
1470 				sounlock(so);
1471 				splx(s);
1472 				return 0;
1473 			}
1474 			if ((m = so->so_rcv.sb_mb) != NULL)
1475 				nextrecord = m->m_nextpkt;
1476 		}
1477 	}
1478 
1479 	if (m && atomic) {
1480 		flags |= MSG_TRUNC;
1481 		if ((flags & MSG_PEEK) == 0)
1482 			(void) sbdroprecord(&so->so_rcv);
1483 	}
1484 	if ((flags & MSG_PEEK) == 0) {
1485 		if (m == NULL) {
1486 			/*
1487 			 * First part is an inline SB_EMPTY_FIXUP().  Second
1488 			 * part makes sure sb_lastrecord is up-to-date if
1489 			 * there is still data in the socket buffer.
1490 			 */
1491 			so->so_rcv.sb_mb = nextrecord;
1492 			if (so->so_rcv.sb_mb == NULL) {
1493 				so->so_rcv.sb_mbtail = NULL;
1494 				so->so_rcv.sb_lastrecord = NULL;
1495 			} else if (nextrecord->m_nextpkt == NULL)
1496 				so->so_rcv.sb_lastrecord = nextrecord;
1497 		}
1498 		SBLASTRECORDCHK(&so->so_rcv, "soreceive 4");
1499 		SBLASTMBUFCHK(&so->so_rcv, "soreceive 4");
1500 		if (pr->pr_flags & PR_WANTRCVD && so->so_pcb)
1501 			(*pr->pr_usrreq)(so, PRU_RCVD, NULL,
1502 			    (struct mbuf *)(long)flags, NULL, l);
1503 	}
1504 	if (orig_resid == uio->uio_resid && orig_resid &&
1505 	    (flags & MSG_EOR) == 0 && (so->so_state & SS_CANTRCVMORE) == 0) {
1506 		sbunlock(&so->so_rcv);
1507 		goto restart;
1508 	}
1509 
1510 	if (flagsp != NULL)
1511 		*flagsp |= flags;
1512  release:
1513 	sbunlock(&so->so_rcv);
1514 	sounlock(so);
1515 	splx(s);
1516 	return error;
1517 }
1518 
1519 int
1520 soshutdown(struct socket *so, int how)
1521 {
1522 	const struct protosw	*pr;
1523 	int	error;
1524 
1525 	KASSERT(solocked(so));
1526 
1527 	pr = so->so_proto;
1528 	if (!(how == SHUT_RD || how == SHUT_WR || how == SHUT_RDWR))
1529 		return (EINVAL);
1530 
1531 	if (how == SHUT_RD || how == SHUT_RDWR) {
1532 		sorflush(so);
1533 		error = 0;
1534 	}
1535 	if (how == SHUT_WR || how == SHUT_RDWR)
1536 		error = (*pr->pr_usrreq)(so, PRU_SHUTDOWN, NULL,
1537 		    NULL, NULL, NULL);
1538 
1539 	return error;
1540 }
1541 
1542 void
1543 sorflush(struct socket *so)
1544 {
1545 	struct sockbuf	*sb, asb;
1546 	const struct protosw	*pr;
1547 
1548 	KASSERT(solocked(so));
1549 
1550 	sb = &so->so_rcv;
1551 	pr = so->so_proto;
1552 	socantrcvmore(so);
1553 	sb->sb_flags |= SB_NOINTR;
1554 	(void )sblock(sb, M_WAITOK);
1555 	sbunlock(sb);
1556 	asb = *sb;
1557 	/*
1558 	 * Clear most of the sockbuf structure, but leave some of the
1559 	 * fields valid.
1560 	 */
1561 	memset(&sb->sb_startzero, 0,
1562 	    sizeof(*sb) - offsetof(struct sockbuf, sb_startzero));
1563 	if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose) {
1564 		sounlock(so);
1565 		(*pr->pr_domain->dom_dispose)(asb.sb_mb);
1566 		solock(so);
1567 	}
1568 	sbrelease(&asb, so);
1569 }
1570 
1571 static int
1572 sosetopt1(struct socket *so, int level, int optname, struct mbuf *m)
1573 {
1574 	int optval, val;
1575 	struct linger	*l;
1576 	struct sockbuf	*sb;
1577 	struct timeval *tv;
1578 
1579 	switch (optname) {
1580 
1581 	case SO_LINGER:
1582 		if (m == NULL || m->m_len != sizeof(struct linger))
1583 			return EINVAL;
1584 		l = mtod(m, struct linger *);
1585 		if (l->l_linger < 0 || l->l_linger > USHRT_MAX ||
1586 		    l->l_linger > (INT_MAX / hz))
1587 			return EDOM;
1588 		so->so_linger = l->l_linger;
1589 		if (l->l_onoff)
1590 			so->so_options |= SO_LINGER;
1591 		else
1592 			so->so_options &= ~SO_LINGER;
1593 		break;
1594 
1595 	case SO_DEBUG:
1596 	case SO_KEEPALIVE:
1597 	case SO_DONTROUTE:
1598 	case SO_USELOOPBACK:
1599 	case SO_BROADCAST:
1600 	case SO_REUSEADDR:
1601 	case SO_REUSEPORT:
1602 	case SO_OOBINLINE:
1603 	case SO_TIMESTAMP:
1604 		if (m == NULL || m->m_len < sizeof(int))
1605 			return EINVAL;
1606 		if (*mtod(m, int *))
1607 			so->so_options |= optname;
1608 		else
1609 			so->so_options &= ~optname;
1610 		break;
1611 
1612 	case SO_SNDBUF:
1613 	case SO_RCVBUF:
1614 	case SO_SNDLOWAT:
1615 	case SO_RCVLOWAT:
1616 		if (m == NULL || m->m_len < sizeof(int))
1617 			return EINVAL;
1618 
1619 		/*
1620 		 * Values < 1 make no sense for any of these
1621 		 * options, so disallow them.
1622 		 */
1623 		optval = *mtod(m, int *);
1624 		if (optval < 1)
1625 			return EINVAL;
1626 
1627 		switch (optname) {
1628 
1629 		case SO_SNDBUF:
1630 		case SO_RCVBUF:
1631 			sb = (optname == SO_SNDBUF) ?
1632 			    &so->so_snd : &so->so_rcv;
1633 			if (sbreserve(sb, (u_long)optval, so) == 0)
1634 				return ENOBUFS;
1635 			sb->sb_flags &= ~SB_AUTOSIZE;
1636 			break;
1637 
1638 		/*
1639 		 * Make sure the low-water is never greater than
1640 		 * the high-water.
1641 		 */
1642 		case SO_SNDLOWAT:
1643 			so->so_snd.sb_lowat =
1644 			    (optval > so->so_snd.sb_hiwat) ?
1645 			    so->so_snd.sb_hiwat : optval;
1646 			break;
1647 		case SO_RCVLOWAT:
1648 			so->so_rcv.sb_lowat =
1649 			    (optval > so->so_rcv.sb_hiwat) ?
1650 			    so->so_rcv.sb_hiwat : optval;
1651 			break;
1652 		}
1653 		break;
1654 
1655 	case SO_SNDTIMEO:
1656 	case SO_RCVTIMEO:
1657 		if (m == NULL || m->m_len < sizeof(*tv))
1658 			return EINVAL;
1659 		tv = mtod(m, struct timeval *);
1660 		if (tv->tv_sec > (INT_MAX - tv->tv_usec / tick) / hz)
1661 			return EDOM;
1662 		val = tv->tv_sec * hz + tv->tv_usec / tick;
1663 		if (val == 0 && tv->tv_usec != 0)
1664 			val = 1;
1665 
1666 		switch (optname) {
1667 
1668 		case SO_SNDTIMEO:
1669 			so->so_snd.sb_timeo = val;
1670 			break;
1671 		case SO_RCVTIMEO:
1672 			so->so_rcv.sb_timeo = val;
1673 			break;
1674 		}
1675 		break;
1676 
1677 	default:
1678 		return ENOPROTOOPT;
1679 	}
1680 	return 0;
1681 }
1682 
1683 int
1684 sosetopt(struct socket *so, int level, int optname, struct mbuf *m)
1685 {
1686 	int error, prerr;
1687 
1688 	solock(so);
1689 	if (level == SOL_SOCKET)
1690 		error = sosetopt1(so, level, optname, m);
1691 	else
1692 		error = ENOPROTOOPT;
1693 
1694 	if ((error == 0 || error == ENOPROTOOPT) &&
1695 	    so->so_proto != NULL && so->so_proto->pr_ctloutput != NULL) {
1696 		/* give the protocol stack a shot */
1697 		prerr = (*so->so_proto->pr_ctloutput)(PRCO_SETOPT, so, level,
1698 		    optname, &m);
1699 		if (prerr == 0)
1700 			error = 0;
1701 		else if (prerr != ENOPROTOOPT)
1702 			error = prerr;
1703 	} else if (m != NULL)
1704 		(void)m_free(m);
1705 	sounlock(so);
1706 	return error;
1707 }
1708 
1709 int
1710 sogetopt(struct socket *so, int level, int optname, struct mbuf **mp)
1711 {
1712 	struct mbuf	*m;
1713 	int		error;
1714 
1715 	solock(so);
1716 	if (level != SOL_SOCKET) {
1717 		if (so->so_proto && so->so_proto->pr_ctloutput) {
1718 			error = ((*so->so_proto->pr_ctloutput)
1719 				  (PRCO_GETOPT, so, level, optname, mp));
1720 		} else
1721 			error = (ENOPROTOOPT);
1722 	} else {
1723 		m = m_get(M_WAIT, MT_SOOPTS);
1724 		m->m_len = sizeof(int);
1725 
1726 		switch (optname) {
1727 
1728 		case SO_LINGER:
1729 			m->m_len = sizeof(struct linger);
1730 			mtod(m, struct linger *)->l_onoff =
1731 			    (so->so_options & SO_LINGER) ? 1 : 0;
1732 			mtod(m, struct linger *)->l_linger = so->so_linger;
1733 			break;
1734 
1735 		case SO_USELOOPBACK:
1736 		case SO_DONTROUTE:
1737 		case SO_DEBUG:
1738 		case SO_KEEPALIVE:
1739 		case SO_REUSEADDR:
1740 		case SO_REUSEPORT:
1741 		case SO_BROADCAST:
1742 		case SO_OOBINLINE:
1743 		case SO_TIMESTAMP:
1744 			*mtod(m, int *) = (so->so_options & optname) ? 1 : 0;
1745 			break;
1746 
1747 		case SO_TYPE:
1748 			*mtod(m, int *) = so->so_type;
1749 			break;
1750 
1751 		case SO_ERROR:
1752 			*mtod(m, int *) = so->so_error;
1753 			so->so_error = 0;
1754 			break;
1755 
1756 		case SO_SNDBUF:
1757 			*mtod(m, int *) = so->so_snd.sb_hiwat;
1758 			break;
1759 
1760 		case SO_RCVBUF:
1761 			*mtod(m, int *) = so->so_rcv.sb_hiwat;
1762 			break;
1763 
1764 		case SO_SNDLOWAT:
1765 			*mtod(m, int *) = so->so_snd.sb_lowat;
1766 			break;
1767 
1768 		case SO_RCVLOWAT:
1769 			*mtod(m, int *) = so->so_rcv.sb_lowat;
1770 			break;
1771 
1772 		case SO_SNDTIMEO:
1773 		case SO_RCVTIMEO:
1774 		    {
1775 			int val = (optname == SO_SNDTIMEO ?
1776 			     so->so_snd.sb_timeo : so->so_rcv.sb_timeo);
1777 
1778 			m->m_len = sizeof(struct timeval);
1779 			mtod(m, struct timeval *)->tv_sec = val / hz;
1780 			mtod(m, struct timeval *)->tv_usec =
1781 			    (val % hz) * tick;
1782 			break;
1783 		    }
1784 
1785 		case SO_OVERFLOWED:
1786 			*mtod(m, int *) = so->so_rcv.sb_overflowed;
1787 			break;
1788 
1789 		default:
1790 			sounlock(so);
1791 			(void)m_free(m);
1792 			return (ENOPROTOOPT);
1793 		}
1794 		*mp = m;
1795 		error = 0;
1796 	}
1797 
1798 	sounlock(so);
1799 	return (error);
1800 }
1801 
1802 void
1803 sohasoutofband(struct socket *so)
1804 {
1805 
1806 	fownsignal(so->so_pgid, SIGURG, POLL_PRI, POLLPRI|POLLRDBAND, so);
1807 	selnotify(&so->so_rcv.sb_sel, POLLPRI | POLLRDBAND, 0);
1808 }
1809 
1810 static void
1811 filt_sordetach(struct knote *kn)
1812 {
1813 	struct socket	*so;
1814 
1815 	so = ((file_t *)kn->kn_obj)->f_data;
1816 	solock(so);
1817 	SLIST_REMOVE(&so->so_rcv.sb_sel.sel_klist, kn, knote, kn_selnext);
1818 	if (SLIST_EMPTY(&so->so_rcv.sb_sel.sel_klist))
1819 		so->so_rcv.sb_flags &= ~SB_KNOTE;
1820 	sounlock(so);
1821 }
1822 
1823 /*ARGSUSED*/
1824 static int
1825 filt_soread(struct knote *kn, long hint)
1826 {
1827 	struct socket	*so;
1828 	int rv;
1829 
1830 	so = ((file_t *)kn->kn_obj)->f_data;
1831 	if (hint != NOTE_SUBMIT)
1832 		solock(so);
1833 	kn->kn_data = so->so_rcv.sb_cc;
1834 	if (so->so_state & SS_CANTRCVMORE) {
1835 		kn->kn_flags |= EV_EOF;
1836 		kn->kn_fflags = so->so_error;
1837 		rv = 1;
1838 	} else if (so->so_error)	/* temporary udp error */
1839 		rv = 1;
1840 	else if (kn->kn_sfflags & NOTE_LOWAT)
1841 		rv = (kn->kn_data >= kn->kn_sdata);
1842 	else
1843 		rv = (kn->kn_data >= so->so_rcv.sb_lowat);
1844 	if (hint != NOTE_SUBMIT)
1845 		sounlock(so);
1846 	return rv;
1847 }
1848 
1849 static void
1850 filt_sowdetach(struct knote *kn)
1851 {
1852 	struct socket	*so;
1853 
1854 	so = ((file_t *)kn->kn_obj)->f_data;
1855 	solock(so);
1856 	SLIST_REMOVE(&so->so_snd.sb_sel.sel_klist, kn, knote, kn_selnext);
1857 	if (SLIST_EMPTY(&so->so_snd.sb_sel.sel_klist))
1858 		so->so_snd.sb_flags &= ~SB_KNOTE;
1859 	sounlock(so);
1860 }
1861 
1862 /*ARGSUSED*/
1863 static int
1864 filt_sowrite(struct knote *kn, long hint)
1865 {
1866 	struct socket	*so;
1867 	int rv;
1868 
1869 	so = ((file_t *)kn->kn_obj)->f_data;
1870 	if (hint != NOTE_SUBMIT)
1871 		solock(so);
1872 	kn->kn_data = sbspace(&so->so_snd);
1873 	if (so->so_state & SS_CANTSENDMORE) {
1874 		kn->kn_flags |= EV_EOF;
1875 		kn->kn_fflags = so->so_error;
1876 		rv = 1;
1877 	} else if (so->so_error)	/* temporary udp error */
1878 		rv = 1;
1879 	else if (((so->so_state & SS_ISCONNECTED) == 0) &&
1880 	    (so->so_proto->pr_flags & PR_CONNREQUIRED))
1881 		rv = 0;
1882 	else if (kn->kn_sfflags & NOTE_LOWAT)
1883 		rv = (kn->kn_data >= kn->kn_sdata);
1884 	else
1885 		rv = (kn->kn_data >= so->so_snd.sb_lowat);
1886 	if (hint != NOTE_SUBMIT)
1887 		sounlock(so);
1888 	return rv;
1889 }
1890 
1891 /*ARGSUSED*/
1892 static int
1893 filt_solisten(struct knote *kn, long hint)
1894 {
1895 	struct socket	*so;
1896 	int rv;
1897 
1898 	so = ((file_t *)kn->kn_obj)->f_data;
1899 
1900 	/*
1901 	 * Set kn_data to number of incoming connections, not
1902 	 * counting partial (incomplete) connections.
1903 	 */
1904 	if (hint != NOTE_SUBMIT)
1905 		solock(so);
1906 	kn->kn_data = so->so_qlen;
1907 	rv = (kn->kn_data > 0);
1908 	if (hint != NOTE_SUBMIT)
1909 		sounlock(so);
1910 	return rv;
1911 }
1912 
1913 static const struct filterops solisten_filtops =
1914 	{ 1, NULL, filt_sordetach, filt_solisten };
1915 static const struct filterops soread_filtops =
1916 	{ 1, NULL, filt_sordetach, filt_soread };
1917 static const struct filterops sowrite_filtops =
1918 	{ 1, NULL, filt_sowdetach, filt_sowrite };
1919 
1920 int
1921 soo_kqfilter(struct file *fp, struct knote *kn)
1922 {
1923 	struct socket	*so;
1924 	struct sockbuf	*sb;
1925 
1926 	so = ((file_t *)kn->kn_obj)->f_data;
1927 	solock(so);
1928 	switch (kn->kn_filter) {
1929 	case EVFILT_READ:
1930 		if (so->so_options & SO_ACCEPTCONN)
1931 			kn->kn_fop = &solisten_filtops;
1932 		else
1933 			kn->kn_fop = &soread_filtops;
1934 		sb = &so->so_rcv;
1935 		break;
1936 	case EVFILT_WRITE:
1937 		kn->kn_fop = &sowrite_filtops;
1938 		sb = &so->so_snd;
1939 		break;
1940 	default:
1941 		sounlock(so);
1942 		return (EINVAL);
1943 	}
1944 	SLIST_INSERT_HEAD(&sb->sb_sel.sel_klist, kn, kn_selnext);
1945 	sb->sb_flags |= SB_KNOTE;
1946 	sounlock(so);
1947 	return (0);
1948 }
1949 
1950 static int
1951 sodopoll(struct socket *so, int events)
1952 {
1953 	int revents;
1954 
1955 	revents = 0;
1956 
1957 	if (events & (POLLIN | POLLRDNORM))
1958 		if (soreadable(so))
1959 			revents |= events & (POLLIN | POLLRDNORM);
1960 
1961 	if (events & (POLLOUT | POLLWRNORM))
1962 		if (sowritable(so))
1963 			revents |= events & (POLLOUT | POLLWRNORM);
1964 
1965 	if (events & (POLLPRI | POLLRDBAND))
1966 		if (so->so_oobmark || (so->so_state & SS_RCVATMARK))
1967 			revents |= events & (POLLPRI | POLLRDBAND);
1968 
1969 	return revents;
1970 }
1971 
1972 int
1973 sopoll(struct socket *so, int events)
1974 {
1975 	int revents = 0;
1976 
1977 #ifndef DIAGNOSTIC
1978 	/*
1979 	 * Do a quick, unlocked check in expectation that the socket
1980 	 * will be ready for I/O.  Don't do this check if DIAGNOSTIC,
1981 	 * as the solocked() assertions will fail.
1982 	 */
1983 	if ((revents = sodopoll(so, events)) != 0)
1984 		return revents;
1985 #endif
1986 
1987 	solock(so);
1988 	if ((revents = sodopoll(so, events)) == 0) {
1989 		if (events & (POLLIN | POLLPRI | POLLRDNORM | POLLRDBAND)) {
1990 			selrecord(curlwp, &so->so_rcv.sb_sel);
1991 			so->so_rcv.sb_flags |= SB_NOTIFY;
1992 		}
1993 
1994 		if (events & (POLLOUT | POLLWRNORM)) {
1995 			selrecord(curlwp, &so->so_snd.sb_sel);
1996 			so->so_snd.sb_flags |= SB_NOTIFY;
1997 		}
1998 	}
1999 	sounlock(so);
2000 
2001 	return revents;
2002 }
2003 
2004 
2005 #include <sys/sysctl.h>
2006 
2007 static int sysctl_kern_somaxkva(SYSCTLFN_PROTO);
2008 
2009 /*
2010  * sysctl helper routine for kern.somaxkva.  ensures that the given
2011  * value is not too small.
2012  * (XXX should we maybe make sure it's not too large as well?)
2013  */
2014 static int
2015 sysctl_kern_somaxkva(SYSCTLFN_ARGS)
2016 {
2017 	int error, new_somaxkva;
2018 	struct sysctlnode node;
2019 
2020 	new_somaxkva = somaxkva;
2021 	node = *rnode;
2022 	node.sysctl_data = &new_somaxkva;
2023 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
2024 	if (error || newp == NULL)
2025 		return (error);
2026 
2027 	if (new_somaxkva < (16 * 1024 * 1024)) /* sanity */
2028 		return (EINVAL);
2029 
2030 	mutex_enter(&so_pendfree_lock);
2031 	somaxkva = new_somaxkva;
2032 	cv_broadcast(&socurkva_cv);
2033 	mutex_exit(&so_pendfree_lock);
2034 
2035 	return (error);
2036 }
2037 
2038 SYSCTL_SETUP(sysctl_kern_somaxkva_setup, "sysctl kern.somaxkva setup")
2039 {
2040 
2041 	sysctl_createv(clog, 0, NULL, NULL,
2042 		       CTLFLAG_PERMANENT,
2043 		       CTLTYPE_NODE, "kern", NULL,
2044 		       NULL, 0, NULL, 0,
2045 		       CTL_KERN, CTL_EOL);
2046 
2047 	sysctl_createv(clog, 0, NULL, NULL,
2048 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
2049 		       CTLTYPE_INT, "somaxkva",
2050 		       SYSCTL_DESCR("Maximum amount of kernel memory to be "
2051 				    "used for socket buffers"),
2052 		       sysctl_kern_somaxkva, 0, NULL, 0,
2053 		       CTL_KERN, KERN_SOMAXKVA, CTL_EOL);
2054 }
2055