xref: /netbsd-src/sys/kern/uipc_socket.c (revision e39ef1d61eee3ccba837ee281f1e098c864487aa)
1 /*	$NetBSD: uipc_socket.c,v 1.206 2011/12/20 23:56:28 christos Exp $	*/
2 
3 /*-
4  * Copyright (c) 2002, 2007, 2008, 2009 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Jason R. Thorpe of Wasabi Systems, Inc, and by Andrew Doran.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 /*
33  * Copyright (c) 2004 The FreeBSD Foundation
34  * Copyright (c) 2004 Robert Watson
35  * Copyright (c) 1982, 1986, 1988, 1990, 1993
36  *	The Regents of the University of California.  All rights reserved.
37  *
38  * Redistribution and use in source and binary forms, with or without
39  * modification, are permitted provided that the following conditions
40  * are met:
41  * 1. Redistributions of source code must retain the above copyright
42  *    notice, this list of conditions and the following disclaimer.
43  * 2. Redistributions in binary form must reproduce the above copyright
44  *    notice, this list of conditions and the following disclaimer in the
45  *    documentation and/or other materials provided with the distribution.
46  * 3. Neither the name of the University nor the names of its contributors
47  *    may be used to endorse or promote products derived from this software
48  *    without specific prior written permission.
49  *
50  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
51  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
52  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
53  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
54  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
55  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
56  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
57  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
58  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
59  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
60  * SUCH DAMAGE.
61  *
62  *	@(#)uipc_socket.c	8.6 (Berkeley) 5/2/95
63  */
64 
65 #include <sys/cdefs.h>
66 __KERNEL_RCSID(0, "$NetBSD: uipc_socket.c,v 1.206 2011/12/20 23:56:28 christos Exp $");
67 
68 #include "opt_compat_netbsd.h"
69 #include "opt_sock_counters.h"
70 #include "opt_sosend_loan.h"
71 #include "opt_mbuftrace.h"
72 #include "opt_somaxkva.h"
73 #include "opt_multiprocessor.h"	/* XXX */
74 
75 #include <sys/param.h>
76 #include <sys/systm.h>
77 #include <sys/proc.h>
78 #include <sys/file.h>
79 #include <sys/filedesc.h>
80 #include <sys/kmem.h>
81 #include <sys/mbuf.h>
82 #include <sys/domain.h>
83 #include <sys/kernel.h>
84 #include <sys/protosw.h>
85 #include <sys/socket.h>
86 #include <sys/socketvar.h>
87 #include <sys/signalvar.h>
88 #include <sys/resourcevar.h>
89 #include <sys/uidinfo.h>
90 #include <sys/event.h>
91 #include <sys/poll.h>
92 #include <sys/kauth.h>
93 #include <sys/mutex.h>
94 #include <sys/condvar.h>
95 #include <sys/kthread.h>
96 
97 #ifdef COMPAT_50
98 #include <compat/sys/time.h>
99 #include <compat/sys/socket.h>
100 #endif
101 
102 #include <uvm/uvm_extern.h>
103 #include <uvm/uvm_loan.h>
104 #include <uvm/uvm_page.h>
105 
106 MALLOC_DEFINE(M_SOOPTS, "soopts", "socket options");
107 MALLOC_DEFINE(M_SONAME, "soname", "socket name");
108 
109 extern const struct fileops socketops;
110 
111 extern int	somaxconn;			/* patchable (XXX sysctl) */
112 int		somaxconn = SOMAXCONN;
113 kmutex_t	*softnet_lock;
114 
115 #ifdef SOSEND_COUNTERS
116 #include <sys/device.h>
117 
118 static struct evcnt sosend_loan_big = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
119     NULL, "sosend", "loan big");
120 static struct evcnt sosend_copy_big = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
121     NULL, "sosend", "copy big");
122 static struct evcnt sosend_copy_small = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
123     NULL, "sosend", "copy small");
124 static struct evcnt sosend_kvalimit = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
125     NULL, "sosend", "kva limit");
126 
127 #define	SOSEND_COUNTER_INCR(ev)		(ev)->ev_count++
128 
129 EVCNT_ATTACH_STATIC(sosend_loan_big);
130 EVCNT_ATTACH_STATIC(sosend_copy_big);
131 EVCNT_ATTACH_STATIC(sosend_copy_small);
132 EVCNT_ATTACH_STATIC(sosend_kvalimit);
133 #else
134 
135 #define	SOSEND_COUNTER_INCR(ev)		/* nothing */
136 
137 #endif /* SOSEND_COUNTERS */
138 
139 static struct callback_entry sokva_reclaimerentry;
140 
141 #if defined(SOSEND_NO_LOAN) || defined(MULTIPROCESSOR)
142 int sock_loan_thresh = -1;
143 #else
144 int sock_loan_thresh = 4096;
145 #endif
146 
147 static kmutex_t so_pendfree_lock;
148 static struct mbuf *so_pendfree = NULL;
149 
150 #ifndef SOMAXKVA
151 #define	SOMAXKVA (16 * 1024 * 1024)
152 #endif
153 int somaxkva = SOMAXKVA;
154 static int socurkva;
155 static kcondvar_t socurkva_cv;
156 
157 static kauth_listener_t socket_listener;
158 
159 #define	SOCK_LOAN_CHUNK		65536
160 
161 static void sopendfree_thread(void *);
162 static kcondvar_t pendfree_thread_cv;
163 static lwp_t *sopendfree_lwp;
164 
165 static void sysctl_kern_somaxkva_setup(void);
166 static struct sysctllog *socket_sysctllog;
167 
168 static vsize_t
169 sokvareserve(struct socket *so, vsize_t len)
170 {
171 	int error;
172 
173 	mutex_enter(&so_pendfree_lock);
174 	while (socurkva + len > somaxkva) {
175 		SOSEND_COUNTER_INCR(&sosend_kvalimit);
176 		error = cv_wait_sig(&socurkva_cv, &so_pendfree_lock);
177 		if (error) {
178 			len = 0;
179 			break;
180 		}
181 	}
182 	socurkva += len;
183 	mutex_exit(&so_pendfree_lock);
184 	return len;
185 }
186 
187 static void
188 sokvaunreserve(vsize_t len)
189 {
190 
191 	mutex_enter(&so_pendfree_lock);
192 	socurkva -= len;
193 	cv_broadcast(&socurkva_cv);
194 	mutex_exit(&so_pendfree_lock);
195 }
196 
197 /*
198  * sokvaalloc: allocate kva for loan.
199  */
200 
201 vaddr_t
202 sokvaalloc(vsize_t len, struct socket *so)
203 {
204 	vaddr_t lva;
205 
206 	/*
207 	 * reserve kva.
208 	 */
209 
210 	if (sokvareserve(so, len) == 0)
211 		return 0;
212 
213 	/*
214 	 * allocate kva.
215 	 */
216 
217 	lva = uvm_km_alloc(kernel_map, len, 0, UVM_KMF_VAONLY | UVM_KMF_WAITVA);
218 	if (lva == 0) {
219 		sokvaunreserve(len);
220 		return (0);
221 	}
222 
223 	return lva;
224 }
225 
226 /*
227  * sokvafree: free kva for loan.
228  */
229 
230 void
231 sokvafree(vaddr_t sva, vsize_t len)
232 {
233 
234 	/*
235 	 * free kva.
236 	 */
237 
238 	uvm_km_free(kernel_map, sva, len, UVM_KMF_VAONLY);
239 
240 	/*
241 	 * unreserve kva.
242 	 */
243 
244 	sokvaunreserve(len);
245 }
246 
247 static void
248 sodoloanfree(struct vm_page **pgs, void *buf, size_t size)
249 {
250 	vaddr_t sva, eva;
251 	vsize_t len;
252 	int npgs;
253 
254 	KASSERT(pgs != NULL);
255 
256 	eva = round_page((vaddr_t) buf + size);
257 	sva = trunc_page((vaddr_t) buf);
258 	len = eva - sva;
259 	npgs = len >> PAGE_SHIFT;
260 
261 	pmap_kremove(sva, len);
262 	pmap_update(pmap_kernel());
263 	uvm_unloan(pgs, npgs, UVM_LOAN_TOPAGE);
264 	sokvafree(sva, len);
265 }
266 
267 /*
268  * sopendfree_thread: free mbufs on "pendfree" list.
269  * unlock and relock so_pendfree_lock when freeing mbufs.
270  */
271 
272 static void
273 sopendfree_thread(void *v)
274 {
275 	struct mbuf *m, *next;
276 	size_t rv;
277 
278 	mutex_enter(&so_pendfree_lock);
279 
280 	for (;;) {
281 		rv = 0;
282 		while (so_pendfree != NULL) {
283 			m = so_pendfree;
284 			so_pendfree = NULL;
285 			mutex_exit(&so_pendfree_lock);
286 
287 			for (; m != NULL; m = next) {
288 				next = m->m_next;
289 				KASSERT((~m->m_flags & (M_EXT|M_EXT_PAGES)) == 0);
290 				KASSERT(m->m_ext.ext_refcnt == 0);
291 
292 				rv += m->m_ext.ext_size;
293 				sodoloanfree(m->m_ext.ext_pgs, m->m_ext.ext_buf,
294 				    m->m_ext.ext_size);
295 				pool_cache_put(mb_cache, m);
296 			}
297 
298 			mutex_enter(&so_pendfree_lock);
299 		}
300 		if (rv)
301 			cv_broadcast(&socurkva_cv);
302 		cv_wait(&pendfree_thread_cv, &so_pendfree_lock);
303 	}
304 	panic("sopendfree_thread");
305 	/* NOTREACHED */
306 }
307 
308 void
309 soloanfree(struct mbuf *m, void *buf, size_t size, void *arg)
310 {
311 
312 	KASSERT(m != NULL);
313 
314 	/*
315 	 * postpone freeing mbuf.
316 	 *
317 	 * we can't do it in interrupt context
318 	 * because we need to put kva back to kernel_map.
319 	 */
320 
321 	mutex_enter(&so_pendfree_lock);
322 	m->m_next = so_pendfree;
323 	so_pendfree = m;
324 	cv_signal(&pendfree_thread_cv);
325 	mutex_exit(&so_pendfree_lock);
326 }
327 
328 static long
329 sosend_loan(struct socket *so, struct uio *uio, struct mbuf *m, long space)
330 {
331 	struct iovec *iov = uio->uio_iov;
332 	vaddr_t sva, eva;
333 	vsize_t len;
334 	vaddr_t lva;
335 	int npgs, error;
336 	vaddr_t va;
337 	int i;
338 
339 	if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace))
340 		return (0);
341 
342 	if (iov->iov_len < (size_t) space)
343 		space = iov->iov_len;
344 	if (space > SOCK_LOAN_CHUNK)
345 		space = SOCK_LOAN_CHUNK;
346 
347 	eva = round_page((vaddr_t) iov->iov_base + space);
348 	sva = trunc_page((vaddr_t) iov->iov_base);
349 	len = eva - sva;
350 	npgs = len >> PAGE_SHIFT;
351 
352 	KASSERT(npgs <= M_EXT_MAXPAGES);
353 
354 	lva = sokvaalloc(len, so);
355 	if (lva == 0)
356 		return 0;
357 
358 	error = uvm_loan(&uio->uio_vmspace->vm_map, sva, len,
359 	    m->m_ext.ext_pgs, UVM_LOAN_TOPAGE);
360 	if (error) {
361 		sokvafree(lva, len);
362 		return (0);
363 	}
364 
365 	for (i = 0, va = lva; i < npgs; i++, va += PAGE_SIZE)
366 		pmap_kenter_pa(va, VM_PAGE_TO_PHYS(m->m_ext.ext_pgs[i]),
367 		    VM_PROT_READ, 0);
368 	pmap_update(pmap_kernel());
369 
370 	lva += (vaddr_t) iov->iov_base & PAGE_MASK;
371 
372 	MEXTADD(m, (void *) lva, space, M_MBUF, soloanfree, so);
373 	m->m_flags |= M_EXT_PAGES | M_EXT_ROMAP;
374 
375 	uio->uio_resid -= space;
376 	/* uio_offset not updated, not set/used for write(2) */
377 	uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + space;
378 	uio->uio_iov->iov_len -= space;
379 	if (uio->uio_iov->iov_len == 0) {
380 		uio->uio_iov++;
381 		uio->uio_iovcnt--;
382 	}
383 
384 	return (space);
385 }
386 
387 static int
388 sokva_reclaim_callback(struct callback_entry *ce, void *obj, void *arg)
389 {
390 
391 	KASSERT(ce == &sokva_reclaimerentry);
392 	KASSERT(obj == NULL);
393 
394 	if (!vm_map_starved_p(kernel_map)) {
395 		return CALLBACK_CHAIN_ABORT;
396 	}
397 	return CALLBACK_CHAIN_CONTINUE;
398 }
399 
400 struct mbuf *
401 getsombuf(struct socket *so, int type)
402 {
403 	struct mbuf *m;
404 
405 	m = m_get(M_WAIT, type);
406 	MCLAIM(m, so->so_mowner);
407 	return m;
408 }
409 
410 static int
411 socket_listener_cb(kauth_cred_t cred, kauth_action_t action, void *cookie,
412     void *arg0, void *arg1, void *arg2, void *arg3)
413 {
414 	int result;
415 	enum kauth_network_req req;
416 
417 	result = KAUTH_RESULT_DEFER;
418 	req = (enum kauth_network_req)arg0;
419 
420 	if ((action != KAUTH_NETWORK_SOCKET) &&
421 	    (action != KAUTH_NETWORK_BIND))
422 		return result;
423 
424 	switch (req) {
425 	case KAUTH_REQ_NETWORK_BIND_PORT:
426 		result = KAUTH_RESULT_ALLOW;
427 		break;
428 
429 	case KAUTH_REQ_NETWORK_SOCKET_DROP: {
430 		/* Normal users can only drop their own connections. */
431 		struct socket *so = (struct socket *)arg1;
432 
433 		if (proc_uidmatch(cred, so->so_cred))
434 			result = KAUTH_RESULT_ALLOW;
435 
436 		break;
437 		}
438 
439 	case KAUTH_REQ_NETWORK_SOCKET_OPEN:
440 		/* We allow "raw" routing/bluetooth sockets to anyone. */
441 		if ((u_long)arg1 == PF_ROUTE || (u_long)arg1 == PF_OROUTE
442 		    || (u_long)arg1 == PF_BLUETOOTH) {
443 			result = KAUTH_RESULT_ALLOW;
444 		} else {
445 			/* Privileged, let secmodel handle this. */
446 			if ((u_long)arg2 == SOCK_RAW)
447 				break;
448 		}
449 
450 		result = KAUTH_RESULT_ALLOW;
451 
452 		break;
453 
454 	case KAUTH_REQ_NETWORK_SOCKET_CANSEE:
455 		result = KAUTH_RESULT_ALLOW;
456 
457 		break;
458 
459 	default:
460 		break;
461 	}
462 
463 	return result;
464 }
465 
466 void
467 soinit(void)
468 {
469 
470 	sysctl_kern_somaxkva_setup();
471 
472 	mutex_init(&so_pendfree_lock, MUTEX_DEFAULT, IPL_VM);
473 	softnet_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
474 	cv_init(&socurkva_cv, "sokva");
475 	cv_init(&pendfree_thread_cv, "sopendfr");
476 	soinit2();
477 
478 	/* Set the initial adjusted socket buffer size. */
479 	if (sb_max_set(sb_max))
480 		panic("bad initial sb_max value: %lu", sb_max);
481 
482 	callback_register(&vm_map_to_kernel(kernel_map)->vmk_reclaim_callback,
483 	    &sokva_reclaimerentry, NULL, sokva_reclaim_callback);
484 
485 	socket_listener = kauth_listen_scope(KAUTH_SCOPE_NETWORK,
486 	    socket_listener_cb, NULL);
487 }
488 
489 void
490 soinit1(void)
491 {
492 	int error = kthread_create(PRI_NONE, KTHREAD_MPSAFE, NULL,
493 	    sopendfree_thread, NULL, &sopendfree_lwp, "sopendfree");
494 	if (error)
495 		panic("soinit1 %d", error);
496 }
497 
498 /*
499  * Socket operation routines.
500  * These routines are called by the routines in
501  * sys_socket.c or from a system process, and
502  * implement the semantics of socket operations by
503  * switching out to the protocol specific routines.
504  */
505 /*ARGSUSED*/
506 int
507 socreate(int dom, struct socket **aso, int type, int proto, struct lwp *l,
508 	 struct socket *lockso)
509 {
510 	const struct protosw	*prp;
511 	struct socket	*so;
512 	uid_t		uid;
513 	int		error;
514 	kmutex_t	*lock;
515 
516 	error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_SOCKET,
517 	    KAUTH_REQ_NETWORK_SOCKET_OPEN, KAUTH_ARG(dom), KAUTH_ARG(type),
518 	    KAUTH_ARG(proto));
519 	if (error != 0)
520 		return error;
521 
522 	if (proto)
523 		prp = pffindproto(dom, proto, type);
524 	else
525 		prp = pffindtype(dom, type);
526 	if (prp == NULL) {
527 		/* no support for domain */
528 		if (pffinddomain(dom) == 0)
529 			return EAFNOSUPPORT;
530 		/* no support for socket type */
531 		if (proto == 0 && type != 0)
532 			return EPROTOTYPE;
533 		return EPROTONOSUPPORT;
534 	}
535 	if (prp->pr_usrreq == NULL)
536 		return EPROTONOSUPPORT;
537 	if (prp->pr_type != type)
538 		return EPROTOTYPE;
539 
540 	so = soget(true);
541 	so->so_type = type;
542 	so->so_proto = prp;
543 	so->so_send = sosend;
544 	so->so_receive = soreceive;
545 #ifdef MBUFTRACE
546 	so->so_rcv.sb_mowner = &prp->pr_domain->dom_mowner;
547 	so->so_snd.sb_mowner = &prp->pr_domain->dom_mowner;
548 	so->so_mowner = &prp->pr_domain->dom_mowner;
549 #endif
550 	uid = kauth_cred_geteuid(l->l_cred);
551 	so->so_uidinfo = uid_find(uid);
552 	so->so_cpid = l->l_proc->p_pid;
553 	if (lockso != NULL) {
554 		/* Caller wants us to share a lock. */
555 		lock = lockso->so_lock;
556 		so->so_lock = lock;
557 		mutex_obj_hold(lock);
558 		mutex_enter(lock);
559 	} else {
560 		/* Lock assigned and taken during PRU_ATTACH. */
561 	}
562 	error = (*prp->pr_usrreq)(so, PRU_ATTACH, NULL,
563 	    (struct mbuf *)(long)proto, NULL, l);
564 	KASSERT(solocked(so));
565 	if (error != 0) {
566 		so->so_state |= SS_NOFDREF;
567 		sofree(so);
568 		return error;
569 	}
570 	so->so_cred = kauth_cred_dup(l->l_cred);
571 	sounlock(so);
572 	*aso = so;
573 	return 0;
574 }
575 
576 /* On success, write file descriptor to fdout and return zero.  On
577  * failure, return non-zero; *fdout will be undefined.
578  */
579 int
580 fsocreate(int domain, struct socket **sop, int type, int protocol,
581     struct lwp *l, int *fdout)
582 {
583 	struct socket	*so;
584 	struct file	*fp;
585 	int		fd, error;
586 	int		flags = type & SOCK_FLAGS_MASK;
587 
588 	type &= ~SOCK_FLAGS_MASK;
589 	if ((error = fd_allocfile(&fp, &fd)) != 0)
590 		return error;
591 	fd_set_exclose(l, fd, (flags & SOCK_CLOEXEC) != 0);
592 	fp->f_flag = FREAD|FWRITE|((flags & SOCK_NONBLOCK) ? FNONBLOCK : 0);
593 	fp->f_type = DTYPE_SOCKET;
594 	fp->f_ops = &socketops;
595 	error = socreate(domain, &so, type, protocol, l, NULL);
596 	if (error != 0) {
597 		fd_abort(curproc, fp, fd);
598 	} else {
599 		if (sop != NULL)
600 			*sop = so;
601 		fp->f_data = so;
602 		fd_affix(curproc, fp, fd);
603 		*fdout = fd;
604 	}
605 	return error;
606 }
607 
608 int
609 sofamily(const struct socket *so)
610 {
611 	const struct protosw *pr;
612 	const struct domain *dom;
613 
614 	if ((pr = so->so_proto) == NULL)
615 		return AF_UNSPEC;
616 	if ((dom = pr->pr_domain) == NULL)
617 		return AF_UNSPEC;
618 	return dom->dom_family;
619 }
620 
621 int
622 sobind(struct socket *so, struct mbuf *nam, struct lwp *l)
623 {
624 	int	error;
625 
626 	solock(so);
627 	error = (*so->so_proto->pr_usrreq)(so, PRU_BIND, NULL, nam, NULL, l);
628 	sounlock(so);
629 	return error;
630 }
631 
632 int
633 solisten(struct socket *so, int backlog, struct lwp *l)
634 {
635 	int	error;
636 
637 	solock(so);
638 	if ((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING |
639 	    SS_ISDISCONNECTING)) != 0) {
640 	    	sounlock(so);
641 		return (EOPNOTSUPP);
642 	}
643 	error = (*so->so_proto->pr_usrreq)(so, PRU_LISTEN, NULL,
644 	    NULL, NULL, l);
645 	if (error != 0) {
646 		sounlock(so);
647 		return error;
648 	}
649 	if (TAILQ_EMPTY(&so->so_q))
650 		so->so_options |= SO_ACCEPTCONN;
651 	if (backlog < 0)
652 		backlog = 0;
653 	so->so_qlimit = min(backlog, somaxconn);
654 	sounlock(so);
655 	return 0;
656 }
657 
658 void
659 sofree(struct socket *so)
660 {
661 	u_int refs;
662 
663 	KASSERT(solocked(so));
664 
665 	if (so->so_pcb || (so->so_state & SS_NOFDREF) == 0) {
666 		sounlock(so);
667 		return;
668 	}
669 	if (so->so_head) {
670 		/*
671 		 * We must not decommission a socket that's on the accept(2)
672 		 * queue.  If we do, then accept(2) may hang after select(2)
673 		 * indicated that the listening socket was ready.
674 		 */
675 		if (!soqremque(so, 0)) {
676 			sounlock(so);
677 			return;
678 		}
679 	}
680 	if (so->so_rcv.sb_hiwat)
681 		(void)chgsbsize(so->so_uidinfo, &so->so_rcv.sb_hiwat, 0,
682 		    RLIM_INFINITY);
683 	if (so->so_snd.sb_hiwat)
684 		(void)chgsbsize(so->so_uidinfo, &so->so_snd.sb_hiwat, 0,
685 		    RLIM_INFINITY);
686 	sbrelease(&so->so_snd, so);
687 	KASSERT(!cv_has_waiters(&so->so_cv));
688 	KASSERT(!cv_has_waiters(&so->so_rcv.sb_cv));
689 	KASSERT(!cv_has_waiters(&so->so_snd.sb_cv));
690 	sorflush(so);
691 	refs = so->so_aborting;	/* XXX */
692 	/* Remove acccept filter if one is present. */
693 	if (so->so_accf != NULL)
694 		(void)accept_filt_clear(so);
695 	sounlock(so);
696 	if (refs == 0)		/* XXX */
697 		soput(so);
698 }
699 
700 /*
701  * Close a socket on last file table reference removal.
702  * Initiate disconnect if connected.
703  * Free socket when disconnect complete.
704  */
705 int
706 soclose(struct socket *so)
707 {
708 	struct socket	*so2;
709 	int		error;
710 	int		error2;
711 
712 	error = 0;
713 	solock(so);
714 	if (so->so_options & SO_ACCEPTCONN) {
715 		for (;;) {
716 			if ((so2 = TAILQ_FIRST(&so->so_q0)) != 0) {
717 				KASSERT(solocked2(so, so2));
718 				(void) soqremque(so2, 0);
719 				/* soabort drops the lock. */
720 				(void) soabort(so2);
721 				solock(so);
722 				continue;
723 			}
724 			if ((so2 = TAILQ_FIRST(&so->so_q)) != 0) {
725 				KASSERT(solocked2(so, so2));
726 				(void) soqremque(so2, 1);
727 				/* soabort drops the lock. */
728 				(void) soabort(so2);
729 				solock(so);
730 				continue;
731 			}
732 			break;
733 		}
734 	}
735 	if (so->so_pcb == 0)
736 		goto discard;
737 	if (so->so_state & SS_ISCONNECTED) {
738 		if ((so->so_state & SS_ISDISCONNECTING) == 0) {
739 			error = sodisconnect(so);
740 			if (error)
741 				goto drop;
742 		}
743 		if (so->so_options & SO_LINGER) {
744 			if ((so->so_state & (SS_ISDISCONNECTING|SS_NBIO)) ==
745 			    (SS_ISDISCONNECTING|SS_NBIO))
746 				goto drop;
747 			while (so->so_state & SS_ISCONNECTED) {
748 				error = sowait(so, true, so->so_linger * hz);
749 				if (error)
750 					break;
751 			}
752 		}
753 	}
754  drop:
755 	if (so->so_pcb) {
756 		error2 = (*so->so_proto->pr_usrreq)(so, PRU_DETACH,
757 		    NULL, NULL, NULL, NULL);
758 		if (error == 0)
759 			error = error2;
760 	}
761  discard:
762 	if (so->so_state & SS_NOFDREF)
763 		panic("soclose: NOFDREF");
764 	kauth_cred_free(so->so_cred);
765 	so->so_state |= SS_NOFDREF;
766 	sofree(so);
767 	return (error);
768 }
769 
770 /*
771  * Must be called with the socket locked..  Will return with it unlocked.
772  */
773 int
774 soabort(struct socket *so)
775 {
776 	u_int refs;
777 	int error;
778 
779 	KASSERT(solocked(so));
780 	KASSERT(so->so_head == NULL);
781 
782 	so->so_aborting++;		/* XXX */
783 	error = (*so->so_proto->pr_usrreq)(so, PRU_ABORT, NULL,
784 	    NULL, NULL, NULL);
785 	refs = --so->so_aborting;	/* XXX */
786 	if (error || (refs == 0)) {
787 		sofree(so);
788 	} else {
789 		sounlock(so);
790 	}
791 	return error;
792 }
793 
794 int
795 soaccept(struct socket *so, struct mbuf *nam)
796 {
797 	int	error;
798 
799 	KASSERT(solocked(so));
800 
801 	error = 0;
802 	if ((so->so_state & SS_NOFDREF) == 0)
803 		panic("soaccept: !NOFDREF");
804 	so->so_state &= ~SS_NOFDREF;
805 	if ((so->so_state & SS_ISDISCONNECTED) == 0 ||
806 	    (so->so_proto->pr_flags & PR_ABRTACPTDIS) == 0)
807 		error = (*so->so_proto->pr_usrreq)(so, PRU_ACCEPT,
808 		    NULL, nam, NULL, NULL);
809 	else
810 		error = ECONNABORTED;
811 
812 	return (error);
813 }
814 
815 int
816 soconnect(struct socket *so, struct mbuf *nam, struct lwp *l)
817 {
818 	int		error;
819 
820 	KASSERT(solocked(so));
821 
822 	if (so->so_options & SO_ACCEPTCONN)
823 		return (EOPNOTSUPP);
824 	/*
825 	 * If protocol is connection-based, can only connect once.
826 	 * Otherwise, if connected, try to disconnect first.
827 	 * This allows user to disconnect by connecting to, e.g.,
828 	 * a null address.
829 	 */
830 	if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) &&
831 	    ((so->so_proto->pr_flags & PR_CONNREQUIRED) ||
832 	    (error = sodisconnect(so))))
833 		error = EISCONN;
834 	else
835 		error = (*so->so_proto->pr_usrreq)(so, PRU_CONNECT,
836 		    NULL, nam, NULL, l);
837 	return (error);
838 }
839 
840 int
841 soconnect2(struct socket *so1, struct socket *so2)
842 {
843 	int	error;
844 
845 	KASSERT(solocked2(so1, so2));
846 
847 	error = (*so1->so_proto->pr_usrreq)(so1, PRU_CONNECT2,
848 	    NULL, (struct mbuf *)so2, NULL, NULL);
849 	return (error);
850 }
851 
852 int
853 sodisconnect(struct socket *so)
854 {
855 	int	error;
856 
857 	KASSERT(solocked(so));
858 
859 	if ((so->so_state & SS_ISCONNECTED) == 0) {
860 		error = ENOTCONN;
861 	} else if (so->so_state & SS_ISDISCONNECTING) {
862 		error = EALREADY;
863 	} else {
864 		error = (*so->so_proto->pr_usrreq)(so, PRU_DISCONNECT,
865 		    NULL, NULL, NULL, NULL);
866 	}
867 	return (error);
868 }
869 
870 #define	SBLOCKWAIT(f)	(((f) & MSG_DONTWAIT) ? M_NOWAIT : M_WAITOK)
871 /*
872  * Send on a socket.
873  * If send must go all at once and message is larger than
874  * send buffering, then hard error.
875  * Lock against other senders.
876  * If must go all at once and not enough room now, then
877  * inform user that this would block and do nothing.
878  * Otherwise, if nonblocking, send as much as possible.
879  * The data to be sent is described by "uio" if nonzero,
880  * otherwise by the mbuf chain "top" (which must be null
881  * if uio is not).  Data provided in mbuf chain must be small
882  * enough to send all at once.
883  *
884  * Returns nonzero on error, timeout or signal; callers
885  * must check for short counts if EINTR/ERESTART are returned.
886  * Data and control buffers are freed on return.
887  */
888 int
889 sosend(struct socket *so, struct mbuf *addr, struct uio *uio, struct mbuf *top,
890 	struct mbuf *control, int flags, struct lwp *l)
891 {
892 	struct mbuf	**mp, *m;
893 	struct proc	*p;
894 	long		space, len, resid, clen, mlen;
895 	int		error, s, dontroute, atomic;
896 	short		wakeup_state = 0;
897 
898 	p = l->l_proc;
899 	clen = 0;
900 
901 	/*
902 	 * solock() provides atomicity of access.  splsoftnet() prevents
903 	 * protocol processing soft interrupts from interrupting us and
904 	 * blocking (expensive).
905 	 */
906 	s = splsoftnet();
907 	solock(so);
908 	atomic = sosendallatonce(so) || top;
909 	if (uio)
910 		resid = uio->uio_resid;
911 	else
912 		resid = top->m_pkthdr.len;
913 	/*
914 	 * In theory resid should be unsigned.
915 	 * However, space must be signed, as it might be less than 0
916 	 * if we over-committed, and we must use a signed comparison
917 	 * of space and resid.  On the other hand, a negative resid
918 	 * causes us to loop sending 0-length segments to the protocol.
919 	 */
920 	if (resid < 0) {
921 		error = EINVAL;
922 		goto out;
923 	}
924 	dontroute =
925 	    (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 &&
926 	    (so->so_proto->pr_flags & PR_ATOMIC);
927 	l->l_ru.ru_msgsnd++;
928 	if (control)
929 		clen = control->m_len;
930  restart:
931 	if ((error = sblock(&so->so_snd, SBLOCKWAIT(flags))) != 0)
932 		goto out;
933 	do {
934 		if (so->so_state & SS_CANTSENDMORE) {
935 			error = EPIPE;
936 			goto release;
937 		}
938 		if (so->so_error) {
939 			error = so->so_error;
940 			so->so_error = 0;
941 			goto release;
942 		}
943 		if ((so->so_state & SS_ISCONNECTED) == 0) {
944 			if (so->so_proto->pr_flags & PR_CONNREQUIRED) {
945 				if ((so->so_state & SS_ISCONFIRMING) == 0 &&
946 				    !(resid == 0 && clen != 0)) {
947 					error = ENOTCONN;
948 					goto release;
949 				}
950 			} else if (addr == 0) {
951 				error = EDESTADDRREQ;
952 				goto release;
953 			}
954 		}
955 		space = sbspace(&so->so_snd);
956 		if (flags & MSG_OOB)
957 			space += 1024;
958 		if ((atomic && resid > so->so_snd.sb_hiwat) ||
959 		    clen > so->so_snd.sb_hiwat) {
960 			error = EMSGSIZE;
961 			goto release;
962 		}
963 		if (space < resid + clen &&
964 		    (atomic || space < so->so_snd.sb_lowat || space < clen)) {
965 			if ((so->so_state & SS_NBIO) || (flags & MSG_NBIO)) {
966 				error = EWOULDBLOCK;
967 				goto release;
968 			}
969 			sbunlock(&so->so_snd);
970 			if (wakeup_state & SS_RESTARTSYS) {
971 				error = ERESTART;
972 				goto out;
973 			}
974 			error = sbwait(&so->so_snd);
975 			if (error)
976 				goto out;
977 			wakeup_state = so->so_state;
978 			goto restart;
979 		}
980 		wakeup_state = 0;
981 		mp = &top;
982 		space -= clen;
983 		do {
984 			if (uio == NULL) {
985 				/*
986 				 * Data is prepackaged in "top".
987 				 */
988 				resid = 0;
989 				if (flags & MSG_EOR)
990 					top->m_flags |= M_EOR;
991 			} else do {
992 				sounlock(so);
993 				splx(s);
994 				if (top == NULL) {
995 					m = m_gethdr(M_WAIT, MT_DATA);
996 					mlen = MHLEN;
997 					m->m_pkthdr.len = 0;
998 					m->m_pkthdr.rcvif = NULL;
999 				} else {
1000 					m = m_get(M_WAIT, MT_DATA);
1001 					mlen = MLEN;
1002 				}
1003 				MCLAIM(m, so->so_snd.sb_mowner);
1004 				if (sock_loan_thresh >= 0 &&
1005 				    uio->uio_iov->iov_len >= sock_loan_thresh &&
1006 				    space >= sock_loan_thresh &&
1007 				    (len = sosend_loan(so, uio, m,
1008 						       space)) != 0) {
1009 					SOSEND_COUNTER_INCR(&sosend_loan_big);
1010 					space -= len;
1011 					goto have_data;
1012 				}
1013 				if (resid >= MINCLSIZE && space >= MCLBYTES) {
1014 					SOSEND_COUNTER_INCR(&sosend_copy_big);
1015 					m_clget(m, M_DONTWAIT);
1016 					if ((m->m_flags & M_EXT) == 0)
1017 						goto nopages;
1018 					mlen = MCLBYTES;
1019 					if (atomic && top == 0) {
1020 						len = lmin(MCLBYTES - max_hdr,
1021 						    resid);
1022 						m->m_data += max_hdr;
1023 					} else
1024 						len = lmin(MCLBYTES, resid);
1025 					space -= len;
1026 				} else {
1027  nopages:
1028 					SOSEND_COUNTER_INCR(&sosend_copy_small);
1029 					len = lmin(lmin(mlen, resid), space);
1030 					space -= len;
1031 					/*
1032 					 * For datagram protocols, leave room
1033 					 * for protocol headers in first mbuf.
1034 					 */
1035 					if (atomic && top == 0 && len < mlen)
1036 						MH_ALIGN(m, len);
1037 				}
1038 				error = uiomove(mtod(m, void *), (int)len, uio);
1039  have_data:
1040 				resid = uio->uio_resid;
1041 				m->m_len = len;
1042 				*mp = m;
1043 				top->m_pkthdr.len += len;
1044 				s = splsoftnet();
1045 				solock(so);
1046 				if (error != 0)
1047 					goto release;
1048 				mp = &m->m_next;
1049 				if (resid <= 0) {
1050 					if (flags & MSG_EOR)
1051 						top->m_flags |= M_EOR;
1052 					break;
1053 				}
1054 			} while (space > 0 && atomic);
1055 
1056 			if (so->so_state & SS_CANTSENDMORE) {
1057 				error = EPIPE;
1058 				goto release;
1059 			}
1060 			if (dontroute)
1061 				so->so_options |= SO_DONTROUTE;
1062 			if (resid > 0)
1063 				so->so_state |= SS_MORETOCOME;
1064 			error = (*so->so_proto->pr_usrreq)(so,
1065 			    (flags & MSG_OOB) ? PRU_SENDOOB : PRU_SEND,
1066 			    top, addr, control, curlwp);
1067 			if (dontroute)
1068 				so->so_options &= ~SO_DONTROUTE;
1069 			if (resid > 0)
1070 				so->so_state &= ~SS_MORETOCOME;
1071 			clen = 0;
1072 			control = NULL;
1073 			top = NULL;
1074 			mp = &top;
1075 			if (error != 0)
1076 				goto release;
1077 		} while (resid && space > 0);
1078 	} while (resid);
1079 
1080  release:
1081 	sbunlock(&so->so_snd);
1082  out:
1083 	sounlock(so);
1084 	splx(s);
1085 	if (top)
1086 		m_freem(top);
1087 	if (control)
1088 		m_freem(control);
1089 	return (error);
1090 }
1091 
1092 /*
1093  * Following replacement or removal of the first mbuf on the first
1094  * mbuf chain of a socket buffer, push necessary state changes back
1095  * into the socket buffer so that other consumers see the values
1096  * consistently.  'nextrecord' is the callers locally stored value of
1097  * the original value of sb->sb_mb->m_nextpkt which must be restored
1098  * when the lead mbuf changes.  NOTE: 'nextrecord' may be NULL.
1099  */
1100 static void
1101 sbsync(struct sockbuf *sb, struct mbuf *nextrecord)
1102 {
1103 
1104 	KASSERT(solocked(sb->sb_so));
1105 
1106 	/*
1107 	 * First, update for the new value of nextrecord.  If necessary,
1108 	 * make it the first record.
1109 	 */
1110 	if (sb->sb_mb != NULL)
1111 		sb->sb_mb->m_nextpkt = nextrecord;
1112 	else
1113 		sb->sb_mb = nextrecord;
1114 
1115         /*
1116          * Now update any dependent socket buffer fields to reflect
1117          * the new state.  This is an inline of SB_EMPTY_FIXUP, with
1118          * the addition of a second clause that takes care of the
1119          * case where sb_mb has been updated, but remains the last
1120          * record.
1121          */
1122         if (sb->sb_mb == NULL) {
1123                 sb->sb_mbtail = NULL;
1124                 sb->sb_lastrecord = NULL;
1125         } else if (sb->sb_mb->m_nextpkt == NULL)
1126                 sb->sb_lastrecord = sb->sb_mb;
1127 }
1128 
1129 /*
1130  * Implement receive operations on a socket.
1131  * We depend on the way that records are added to the sockbuf
1132  * by sbappend*.  In particular, each record (mbufs linked through m_next)
1133  * must begin with an address if the protocol so specifies,
1134  * followed by an optional mbuf or mbufs containing ancillary data,
1135  * and then zero or more mbufs of data.
1136  * In order to avoid blocking network interrupts for the entire time here,
1137  * we splx() while doing the actual copy to user space.
1138  * Although the sockbuf is locked, new data may still be appended,
1139  * and thus we must maintain consistency of the sockbuf during that time.
1140  *
1141  * The caller may receive the data as a single mbuf chain by supplying
1142  * an mbuf **mp0 for use in returning the chain.  The uio is then used
1143  * only for the count in uio_resid.
1144  */
1145 int
1146 soreceive(struct socket *so, struct mbuf **paddr, struct uio *uio,
1147 	struct mbuf **mp0, struct mbuf **controlp, int *flagsp)
1148 {
1149 	struct lwp *l = curlwp;
1150 	struct mbuf	*m, **mp, *mt;
1151 	int atomic, flags, len, error, s, offset, moff, type, orig_resid;
1152 	const struct protosw	*pr;
1153 	struct mbuf	*nextrecord;
1154 	int		mbuf_removed = 0;
1155 	const struct domain *dom;
1156 	short		wakeup_state = 0;
1157 
1158 	pr = so->so_proto;
1159 	atomic = pr->pr_flags & PR_ATOMIC;
1160 	dom = pr->pr_domain;
1161 	mp = mp0;
1162 	type = 0;
1163 	orig_resid = uio->uio_resid;
1164 
1165 	if (paddr != NULL)
1166 		*paddr = NULL;
1167 	if (controlp != NULL)
1168 		*controlp = NULL;
1169 	if (flagsp != NULL)
1170 		flags = *flagsp &~ MSG_EOR;
1171 	else
1172 		flags = 0;
1173 
1174 	if (flags & MSG_OOB) {
1175 		m = m_get(M_WAIT, MT_DATA);
1176 		solock(so);
1177 		error = (*pr->pr_usrreq)(so, PRU_RCVOOB, m,
1178 		    (struct mbuf *)(long)(flags & MSG_PEEK), NULL, l);
1179 		sounlock(so);
1180 		if (error)
1181 			goto bad;
1182 		do {
1183 			error = uiomove(mtod(m, void *),
1184 			    (int) min(uio->uio_resid, m->m_len), uio);
1185 			m = m_free(m);
1186 		} while (uio->uio_resid > 0 && error == 0 && m);
1187  bad:
1188 		if (m != NULL)
1189 			m_freem(m);
1190 		return error;
1191 	}
1192 	if (mp != NULL)
1193 		*mp = NULL;
1194 
1195 	/*
1196 	 * solock() provides atomicity of access.  splsoftnet() prevents
1197 	 * protocol processing soft interrupts from interrupting us and
1198 	 * blocking (expensive).
1199 	 */
1200 	s = splsoftnet();
1201 	solock(so);
1202 	if (so->so_state & SS_ISCONFIRMING && uio->uio_resid)
1203 		(*pr->pr_usrreq)(so, PRU_RCVD, NULL, NULL, NULL, l);
1204 
1205  restart:
1206 	if ((error = sblock(&so->so_rcv, SBLOCKWAIT(flags))) != 0) {
1207 		sounlock(so);
1208 		splx(s);
1209 		return error;
1210 	}
1211 
1212 	m = so->so_rcv.sb_mb;
1213 	/*
1214 	 * If we have less data than requested, block awaiting more
1215 	 * (subject to any timeout) if:
1216 	 *   1. the current count is less than the low water mark,
1217 	 *   2. MSG_WAITALL is set, and it is possible to do the entire
1218 	 *	receive operation at once if we block (resid <= hiwat), or
1219 	 *   3. MSG_DONTWAIT is not set.
1220 	 * If MSG_WAITALL is set but resid is larger than the receive buffer,
1221 	 * we have to do the receive in sections, and thus risk returning
1222 	 * a short count if a timeout or signal occurs after we start.
1223 	 */
1224 	if (m == NULL ||
1225 	    ((flags & MSG_DONTWAIT) == 0 &&
1226 	     so->so_rcv.sb_cc < uio->uio_resid &&
1227 	     (so->so_rcv.sb_cc < so->so_rcv.sb_lowat ||
1228 	      ((flags & MSG_WAITALL) &&
1229 	       uio->uio_resid <= so->so_rcv.sb_hiwat)) &&
1230 	     m->m_nextpkt == NULL && !atomic)) {
1231 #ifdef DIAGNOSTIC
1232 		if (m == NULL && so->so_rcv.sb_cc)
1233 			panic("receive 1");
1234 #endif
1235 		if (so->so_error) {
1236 			if (m != NULL)
1237 				goto dontblock;
1238 			error = so->so_error;
1239 			if ((flags & MSG_PEEK) == 0)
1240 				so->so_error = 0;
1241 			goto release;
1242 		}
1243 		if (so->so_state & SS_CANTRCVMORE) {
1244 			if (m != NULL)
1245 				goto dontblock;
1246 			else
1247 				goto release;
1248 		}
1249 		for (; m != NULL; m = m->m_next)
1250 			if (m->m_type == MT_OOBDATA  || (m->m_flags & M_EOR)) {
1251 				m = so->so_rcv.sb_mb;
1252 				goto dontblock;
1253 			}
1254 		if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 &&
1255 		    (so->so_proto->pr_flags & PR_CONNREQUIRED)) {
1256 			error = ENOTCONN;
1257 			goto release;
1258 		}
1259 		if (uio->uio_resid == 0)
1260 			goto release;
1261 		if ((so->so_state & SS_NBIO) ||
1262 		    (flags & (MSG_DONTWAIT|MSG_NBIO))) {
1263 			error = EWOULDBLOCK;
1264 			goto release;
1265 		}
1266 		SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 1");
1267 		SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 1");
1268 		sbunlock(&so->so_rcv);
1269 		if (wakeup_state & SS_RESTARTSYS)
1270 			error = ERESTART;
1271 		else
1272 			error = sbwait(&so->so_rcv);
1273 		if (error != 0) {
1274 			sounlock(so);
1275 			splx(s);
1276 			return error;
1277 		}
1278 		wakeup_state = so->so_state;
1279 		goto restart;
1280 	}
1281  dontblock:
1282 	/*
1283 	 * On entry here, m points to the first record of the socket buffer.
1284 	 * From this point onward, we maintain 'nextrecord' as a cache of the
1285 	 * pointer to the next record in the socket buffer.  We must keep the
1286 	 * various socket buffer pointers and local stack versions of the
1287 	 * pointers in sync, pushing out modifications before dropping the
1288 	 * socket lock, and re-reading them when picking it up.
1289 	 *
1290 	 * Otherwise, we will race with the network stack appending new data
1291 	 * or records onto the socket buffer by using inconsistent/stale
1292 	 * versions of the field, possibly resulting in socket buffer
1293 	 * corruption.
1294 	 *
1295 	 * By holding the high-level sblock(), we prevent simultaneous
1296 	 * readers from pulling off the front of the socket buffer.
1297 	 */
1298 	if (l != NULL)
1299 		l->l_ru.ru_msgrcv++;
1300 	KASSERT(m == so->so_rcv.sb_mb);
1301 	SBLASTRECORDCHK(&so->so_rcv, "soreceive 1");
1302 	SBLASTMBUFCHK(&so->so_rcv, "soreceive 1");
1303 	nextrecord = m->m_nextpkt;
1304 	if (pr->pr_flags & PR_ADDR) {
1305 #ifdef DIAGNOSTIC
1306 		if (m->m_type != MT_SONAME)
1307 			panic("receive 1a");
1308 #endif
1309 		orig_resid = 0;
1310 		if (flags & MSG_PEEK) {
1311 			if (paddr)
1312 				*paddr = m_copy(m, 0, m->m_len);
1313 			m = m->m_next;
1314 		} else {
1315 			sbfree(&so->so_rcv, m);
1316 			mbuf_removed = 1;
1317 			if (paddr != NULL) {
1318 				*paddr = m;
1319 				so->so_rcv.sb_mb = m->m_next;
1320 				m->m_next = NULL;
1321 				m = so->so_rcv.sb_mb;
1322 			} else {
1323 				MFREE(m, so->so_rcv.sb_mb);
1324 				m = so->so_rcv.sb_mb;
1325 			}
1326 			sbsync(&so->so_rcv, nextrecord);
1327 		}
1328 	}
1329 
1330 	/*
1331 	 * Process one or more MT_CONTROL mbufs present before any data mbufs
1332 	 * in the first mbuf chain on the socket buffer.  If MSG_PEEK, we
1333 	 * just copy the data; if !MSG_PEEK, we call into the protocol to
1334 	 * perform externalization (or freeing if controlp == NULL).
1335 	 */
1336 	if (__predict_false(m != NULL && m->m_type == MT_CONTROL)) {
1337 		struct mbuf *cm = NULL, *cmn;
1338 		struct mbuf **cme = &cm;
1339 
1340 		do {
1341 			if (flags & MSG_PEEK) {
1342 				if (controlp != NULL) {
1343 					*controlp = m_copy(m, 0, m->m_len);
1344 					controlp = &(*controlp)->m_next;
1345 				}
1346 				m = m->m_next;
1347 			} else {
1348 				sbfree(&so->so_rcv, m);
1349 				so->so_rcv.sb_mb = m->m_next;
1350 				m->m_next = NULL;
1351 				*cme = m;
1352 				cme = &(*cme)->m_next;
1353 				m = so->so_rcv.sb_mb;
1354 			}
1355 		} while (m != NULL && m->m_type == MT_CONTROL);
1356 		if ((flags & MSG_PEEK) == 0)
1357 			sbsync(&so->so_rcv, nextrecord);
1358 		for (; cm != NULL; cm = cmn) {
1359 			cmn = cm->m_next;
1360 			cm->m_next = NULL;
1361 			type = mtod(cm, struct cmsghdr *)->cmsg_type;
1362 			if (controlp != NULL) {
1363 				if (dom->dom_externalize != NULL &&
1364 				    type == SCM_RIGHTS) {
1365 					sounlock(so);
1366 					splx(s);
1367 					error = (*dom->dom_externalize)(cm, l,
1368 					    (flags & MSG_CMSG_CLOEXEC) ?
1369 					    O_CLOEXEC : 0);
1370 					s = splsoftnet();
1371 					solock(so);
1372 				}
1373 				*controlp = cm;
1374 				while (*controlp != NULL)
1375 					controlp = &(*controlp)->m_next;
1376 			} else {
1377 				/*
1378 				 * Dispose of any SCM_RIGHTS message that went
1379 				 * through the read path rather than recv.
1380 				 */
1381 				if (dom->dom_dispose != NULL &&
1382 				    type == SCM_RIGHTS) {
1383 				    	sounlock(so);
1384 					(*dom->dom_dispose)(cm);
1385 					solock(so);
1386 				}
1387 				m_freem(cm);
1388 			}
1389 		}
1390 		if (m != NULL)
1391 			nextrecord = so->so_rcv.sb_mb->m_nextpkt;
1392 		else
1393 			nextrecord = so->so_rcv.sb_mb;
1394 		orig_resid = 0;
1395 	}
1396 
1397 	/* If m is non-NULL, we have some data to read. */
1398 	if (__predict_true(m != NULL)) {
1399 		type = m->m_type;
1400 		if (type == MT_OOBDATA)
1401 			flags |= MSG_OOB;
1402 	}
1403 	SBLASTRECORDCHK(&so->so_rcv, "soreceive 2");
1404 	SBLASTMBUFCHK(&so->so_rcv, "soreceive 2");
1405 
1406 	moff = 0;
1407 	offset = 0;
1408 	while (m != NULL && uio->uio_resid > 0 && error == 0) {
1409 		if (m->m_type == MT_OOBDATA) {
1410 			if (type != MT_OOBDATA)
1411 				break;
1412 		} else if (type == MT_OOBDATA)
1413 			break;
1414 #ifdef DIAGNOSTIC
1415 		else if (m->m_type != MT_DATA && m->m_type != MT_HEADER)
1416 			panic("receive 3");
1417 #endif
1418 		so->so_state &= ~SS_RCVATMARK;
1419 		wakeup_state = 0;
1420 		len = uio->uio_resid;
1421 		if (so->so_oobmark && len > so->so_oobmark - offset)
1422 			len = so->so_oobmark - offset;
1423 		if (len > m->m_len - moff)
1424 			len = m->m_len - moff;
1425 		/*
1426 		 * If mp is set, just pass back the mbufs.
1427 		 * Otherwise copy them out via the uio, then free.
1428 		 * Sockbuf must be consistent here (points to current mbuf,
1429 		 * it points to next record) when we drop priority;
1430 		 * we must note any additions to the sockbuf when we
1431 		 * block interrupts again.
1432 		 */
1433 		if (mp == NULL) {
1434 			SBLASTRECORDCHK(&so->so_rcv, "soreceive uiomove");
1435 			SBLASTMBUFCHK(&so->so_rcv, "soreceive uiomove");
1436 			sounlock(so);
1437 			splx(s);
1438 			error = uiomove(mtod(m, char *) + moff, (int)len, uio);
1439 			s = splsoftnet();
1440 			solock(so);
1441 			if (error != 0) {
1442 				/*
1443 				 * If any part of the record has been removed
1444 				 * (such as the MT_SONAME mbuf, which will
1445 				 * happen when PR_ADDR, and thus also
1446 				 * PR_ATOMIC, is set), then drop the entire
1447 				 * record to maintain the atomicity of the
1448 				 * receive operation.
1449 				 *
1450 				 * This avoids a later panic("receive 1a")
1451 				 * when compiled with DIAGNOSTIC.
1452 				 */
1453 				if (m && mbuf_removed && atomic)
1454 					(void) sbdroprecord(&so->so_rcv);
1455 
1456 				goto release;
1457 			}
1458 		} else
1459 			uio->uio_resid -= len;
1460 		if (len == m->m_len - moff) {
1461 			if (m->m_flags & M_EOR)
1462 				flags |= MSG_EOR;
1463 			if (flags & MSG_PEEK) {
1464 				m = m->m_next;
1465 				moff = 0;
1466 			} else {
1467 				nextrecord = m->m_nextpkt;
1468 				sbfree(&so->so_rcv, m);
1469 				if (mp) {
1470 					*mp = m;
1471 					mp = &m->m_next;
1472 					so->so_rcv.sb_mb = m = m->m_next;
1473 					*mp = NULL;
1474 				} else {
1475 					MFREE(m, so->so_rcv.sb_mb);
1476 					m = so->so_rcv.sb_mb;
1477 				}
1478 				/*
1479 				 * If m != NULL, we also know that
1480 				 * so->so_rcv.sb_mb != NULL.
1481 				 */
1482 				KASSERT(so->so_rcv.sb_mb == m);
1483 				if (m) {
1484 					m->m_nextpkt = nextrecord;
1485 					if (nextrecord == NULL)
1486 						so->so_rcv.sb_lastrecord = m;
1487 				} else {
1488 					so->so_rcv.sb_mb = nextrecord;
1489 					SB_EMPTY_FIXUP(&so->so_rcv);
1490 				}
1491 				SBLASTRECORDCHK(&so->so_rcv, "soreceive 3");
1492 				SBLASTMBUFCHK(&so->so_rcv, "soreceive 3");
1493 			}
1494 		} else if (flags & MSG_PEEK)
1495 			moff += len;
1496 		else {
1497 			if (mp != NULL) {
1498 				mt = m_copym(m, 0, len, M_NOWAIT);
1499 				if (__predict_false(mt == NULL)) {
1500 					sounlock(so);
1501 					mt = m_copym(m, 0, len, M_WAIT);
1502 					solock(so);
1503 				}
1504 				*mp = mt;
1505 			}
1506 			m->m_data += len;
1507 			m->m_len -= len;
1508 			so->so_rcv.sb_cc -= len;
1509 		}
1510 		if (so->so_oobmark) {
1511 			if ((flags & MSG_PEEK) == 0) {
1512 				so->so_oobmark -= len;
1513 				if (so->so_oobmark == 0) {
1514 					so->so_state |= SS_RCVATMARK;
1515 					break;
1516 				}
1517 			} else {
1518 				offset += len;
1519 				if (offset == so->so_oobmark)
1520 					break;
1521 			}
1522 		}
1523 		if (flags & MSG_EOR)
1524 			break;
1525 		/*
1526 		 * If the MSG_WAITALL flag is set (for non-atomic socket),
1527 		 * we must not quit until "uio->uio_resid == 0" or an error
1528 		 * termination.  If a signal/timeout occurs, return
1529 		 * with a short count but without error.
1530 		 * Keep sockbuf locked against other readers.
1531 		 */
1532 		while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 &&
1533 		    !sosendallatonce(so) && !nextrecord) {
1534 			if (so->so_error || so->so_state & SS_CANTRCVMORE)
1535 				break;
1536 			/*
1537 			 * If we are peeking and the socket receive buffer is
1538 			 * full, stop since we can't get more data to peek at.
1539 			 */
1540 			if ((flags & MSG_PEEK) && sbspace(&so->so_rcv) <= 0)
1541 				break;
1542 			/*
1543 			 * If we've drained the socket buffer, tell the
1544 			 * protocol in case it needs to do something to
1545 			 * get it filled again.
1546 			 */
1547 			if ((pr->pr_flags & PR_WANTRCVD) && so->so_pcb)
1548 				(*pr->pr_usrreq)(so, PRU_RCVD,
1549 				    NULL, (struct mbuf *)(long)flags, NULL, l);
1550 			SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 2");
1551 			SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 2");
1552 			if (wakeup_state & SS_RESTARTSYS)
1553 				error = ERESTART;
1554 			else
1555 				error = sbwait(&so->so_rcv);
1556 			if (error != 0) {
1557 				sbunlock(&so->so_rcv);
1558 				sounlock(so);
1559 				splx(s);
1560 				return 0;
1561 			}
1562 			if ((m = so->so_rcv.sb_mb) != NULL)
1563 				nextrecord = m->m_nextpkt;
1564 			wakeup_state = so->so_state;
1565 		}
1566 	}
1567 
1568 	if (m && atomic) {
1569 		flags |= MSG_TRUNC;
1570 		if ((flags & MSG_PEEK) == 0)
1571 			(void) sbdroprecord(&so->so_rcv);
1572 	}
1573 	if ((flags & MSG_PEEK) == 0) {
1574 		if (m == NULL) {
1575 			/*
1576 			 * First part is an inline SB_EMPTY_FIXUP().  Second
1577 			 * part makes sure sb_lastrecord is up-to-date if
1578 			 * there is still data in the socket buffer.
1579 			 */
1580 			so->so_rcv.sb_mb = nextrecord;
1581 			if (so->so_rcv.sb_mb == NULL) {
1582 				so->so_rcv.sb_mbtail = NULL;
1583 				so->so_rcv.sb_lastrecord = NULL;
1584 			} else if (nextrecord->m_nextpkt == NULL)
1585 				so->so_rcv.sb_lastrecord = nextrecord;
1586 		}
1587 		SBLASTRECORDCHK(&so->so_rcv, "soreceive 4");
1588 		SBLASTMBUFCHK(&so->so_rcv, "soreceive 4");
1589 		if (pr->pr_flags & PR_WANTRCVD && so->so_pcb)
1590 			(*pr->pr_usrreq)(so, PRU_RCVD, NULL,
1591 			    (struct mbuf *)(long)flags, NULL, l);
1592 	}
1593 	if (orig_resid == uio->uio_resid && orig_resid &&
1594 	    (flags & MSG_EOR) == 0 && (so->so_state & SS_CANTRCVMORE) == 0) {
1595 		sbunlock(&so->so_rcv);
1596 		goto restart;
1597 	}
1598 
1599 	if (flagsp != NULL)
1600 		*flagsp |= flags;
1601  release:
1602 	sbunlock(&so->so_rcv);
1603 	sounlock(so);
1604 	splx(s);
1605 	return error;
1606 }
1607 
1608 int
1609 soshutdown(struct socket *so, int how)
1610 {
1611 	const struct protosw	*pr;
1612 	int	error;
1613 
1614 	KASSERT(solocked(so));
1615 
1616 	pr = so->so_proto;
1617 	if (!(how == SHUT_RD || how == SHUT_WR || how == SHUT_RDWR))
1618 		return (EINVAL);
1619 
1620 	if (how == SHUT_RD || how == SHUT_RDWR) {
1621 		sorflush(so);
1622 		error = 0;
1623 	}
1624 	if (how == SHUT_WR || how == SHUT_RDWR)
1625 		error = (*pr->pr_usrreq)(so, PRU_SHUTDOWN, NULL,
1626 		    NULL, NULL, NULL);
1627 
1628 	return error;
1629 }
1630 
1631 void
1632 sorestart(struct socket *so)
1633 {
1634 	/*
1635 	 * An application has called close() on an fd on which another
1636 	 * of its threads has called a socket system call.
1637 	 * Mark this and wake everyone up, and code that would block again
1638 	 * instead returns ERESTART.
1639 	 * On system call re-entry the fd is validated and EBADF returned.
1640 	 * Any other fd will block again on the 2nd syscall.
1641 	 */
1642 	solock(so);
1643 	so->so_state |= SS_RESTARTSYS;
1644 	cv_broadcast(&so->so_cv);
1645 	cv_broadcast(&so->so_snd.sb_cv);
1646 	cv_broadcast(&so->so_rcv.sb_cv);
1647 	sounlock(so);
1648 }
1649 
1650 void
1651 sorflush(struct socket *so)
1652 {
1653 	struct sockbuf	*sb, asb;
1654 	const struct protosw	*pr;
1655 
1656 	KASSERT(solocked(so));
1657 
1658 	sb = &so->so_rcv;
1659 	pr = so->so_proto;
1660 	socantrcvmore(so);
1661 	sb->sb_flags |= SB_NOINTR;
1662 	(void )sblock(sb, M_WAITOK);
1663 	sbunlock(sb);
1664 	asb = *sb;
1665 	/*
1666 	 * Clear most of the sockbuf structure, but leave some of the
1667 	 * fields valid.
1668 	 */
1669 	memset(&sb->sb_startzero, 0,
1670 	    sizeof(*sb) - offsetof(struct sockbuf, sb_startzero));
1671 	if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose) {
1672 		sounlock(so);
1673 		(*pr->pr_domain->dom_dispose)(asb.sb_mb);
1674 		solock(so);
1675 	}
1676 	sbrelease(&asb, so);
1677 }
1678 
1679 /*
1680  * internal set SOL_SOCKET options
1681  */
1682 static int
1683 sosetopt1(struct socket *so, const struct sockopt *sopt)
1684 {
1685 	int error = EINVAL, optval, opt;
1686 	struct linger l;
1687 	struct timeval tv;
1688 
1689 	switch ((opt = sopt->sopt_name)) {
1690 
1691 	case SO_ACCEPTFILTER:
1692 		error = accept_filt_setopt(so, sopt);
1693 		KASSERT(solocked(so));
1694 		break;
1695 
1696   	case SO_LINGER:
1697  		error = sockopt_get(sopt, &l, sizeof(l));
1698 		solock(so);
1699  		if (error)
1700  			break;
1701  		if (l.l_linger < 0 || l.l_linger > USHRT_MAX ||
1702  		    l.l_linger > (INT_MAX / hz)) {
1703 			error = EDOM;
1704 			break;
1705 		}
1706  		so->so_linger = l.l_linger;
1707  		if (l.l_onoff)
1708  			so->so_options |= SO_LINGER;
1709  		else
1710  			so->so_options &= ~SO_LINGER;
1711    		break;
1712 
1713 	case SO_DEBUG:
1714 	case SO_KEEPALIVE:
1715 	case SO_DONTROUTE:
1716 	case SO_USELOOPBACK:
1717 	case SO_BROADCAST:
1718 	case SO_REUSEADDR:
1719 	case SO_REUSEPORT:
1720 	case SO_OOBINLINE:
1721 	case SO_TIMESTAMP:
1722 #ifdef SO_OTIMESTAMP
1723 	case SO_OTIMESTAMP:
1724 #endif
1725 		error = sockopt_getint(sopt, &optval);
1726 		solock(so);
1727 		if (error)
1728 			break;
1729 		if (optval)
1730 			so->so_options |= opt;
1731 		else
1732 			so->so_options &= ~opt;
1733 		break;
1734 
1735 	case SO_SNDBUF:
1736 	case SO_RCVBUF:
1737 	case SO_SNDLOWAT:
1738 	case SO_RCVLOWAT:
1739 		error = sockopt_getint(sopt, &optval);
1740 		solock(so);
1741 		if (error)
1742 			break;
1743 
1744 		/*
1745 		 * Values < 1 make no sense for any of these
1746 		 * options, so disallow them.
1747 		 */
1748 		if (optval < 1) {
1749 			error = EINVAL;
1750 			break;
1751 		}
1752 
1753 		switch (opt) {
1754 		case SO_SNDBUF:
1755 			if (sbreserve(&so->so_snd, (u_long)optval, so) == 0) {
1756 				error = ENOBUFS;
1757 				break;
1758 			}
1759 			so->so_snd.sb_flags &= ~SB_AUTOSIZE;
1760 			break;
1761 
1762 		case SO_RCVBUF:
1763 			if (sbreserve(&so->so_rcv, (u_long)optval, so) == 0) {
1764 				error = ENOBUFS;
1765 				break;
1766 			}
1767 			so->so_rcv.sb_flags &= ~SB_AUTOSIZE;
1768 			break;
1769 
1770 		/*
1771 		 * Make sure the low-water is never greater than
1772 		 * the high-water.
1773 		 */
1774 		case SO_SNDLOWAT:
1775 			if (optval > so->so_snd.sb_hiwat)
1776 				optval = so->so_snd.sb_hiwat;
1777 
1778 			so->so_snd.sb_lowat = optval;
1779 			break;
1780 
1781 		case SO_RCVLOWAT:
1782 			if (optval > so->so_rcv.sb_hiwat)
1783 				optval = so->so_rcv.sb_hiwat;
1784 
1785 			so->so_rcv.sb_lowat = optval;
1786 			break;
1787 		}
1788 		break;
1789 
1790 #ifdef COMPAT_50
1791 	case SO_OSNDTIMEO:
1792 	case SO_ORCVTIMEO: {
1793 		struct timeval50 otv;
1794 		error = sockopt_get(sopt, &otv, sizeof(otv));
1795 		if (error) {
1796 			solock(so);
1797 			break;
1798 		}
1799 		timeval50_to_timeval(&otv, &tv);
1800 		opt = opt == SO_OSNDTIMEO ? SO_SNDTIMEO : SO_RCVTIMEO;
1801 		error = 0;
1802 		/*FALLTHROUGH*/
1803 	}
1804 #endif /* COMPAT_50 */
1805 
1806 	case SO_SNDTIMEO:
1807 	case SO_RCVTIMEO:
1808 		if (error)
1809 			error = sockopt_get(sopt, &tv, sizeof(tv));
1810 		solock(so);
1811 		if (error)
1812 			break;
1813 
1814 		if (tv.tv_sec > (INT_MAX - tv.tv_usec / tick) / hz) {
1815 			error = EDOM;
1816 			break;
1817 		}
1818 
1819 		optval = tv.tv_sec * hz + tv.tv_usec / tick;
1820 		if (optval == 0 && tv.tv_usec != 0)
1821 			optval = 1;
1822 
1823 		switch (opt) {
1824 		case SO_SNDTIMEO:
1825 			so->so_snd.sb_timeo = optval;
1826 			break;
1827 		case SO_RCVTIMEO:
1828 			so->so_rcv.sb_timeo = optval;
1829 			break;
1830 		}
1831 		break;
1832 
1833 	default:
1834 		solock(so);
1835 		error = ENOPROTOOPT;
1836 		break;
1837 	}
1838 	KASSERT(solocked(so));
1839 	return error;
1840 }
1841 
1842 int
1843 sosetopt(struct socket *so, struct sockopt *sopt)
1844 {
1845 	int error, prerr;
1846 
1847 	if (sopt->sopt_level == SOL_SOCKET) {
1848 		error = sosetopt1(so, sopt);
1849 		KASSERT(solocked(so));
1850 	} else {
1851 		error = ENOPROTOOPT;
1852 		solock(so);
1853 	}
1854 
1855 	if ((error == 0 || error == ENOPROTOOPT) &&
1856 	    so->so_proto != NULL && so->so_proto->pr_ctloutput != NULL) {
1857 		/* give the protocol stack a shot */
1858 		prerr = (*so->so_proto->pr_ctloutput)(PRCO_SETOPT, so, sopt);
1859 		if (prerr == 0)
1860 			error = 0;
1861 		else if (prerr != ENOPROTOOPT)
1862 			error = prerr;
1863 	}
1864 	sounlock(so);
1865 	return error;
1866 }
1867 
1868 /*
1869  * so_setsockopt() is a wrapper providing a sockopt structure for sosetopt()
1870  */
1871 int
1872 so_setsockopt(struct lwp *l, struct socket *so, int level, int name,
1873     const void *val, size_t valsize)
1874 {
1875 	struct sockopt sopt;
1876 	int error;
1877 
1878 	KASSERT(valsize == 0 || val != NULL);
1879 
1880 	sockopt_init(&sopt, level, name, valsize);
1881 	sockopt_set(&sopt, val, valsize);
1882 
1883 	error = sosetopt(so, &sopt);
1884 
1885 	sockopt_destroy(&sopt);
1886 
1887 	return error;
1888 }
1889 
1890 /*
1891  * internal get SOL_SOCKET options
1892  */
1893 static int
1894 sogetopt1(struct socket *so, struct sockopt *sopt)
1895 {
1896 	int error, optval, opt;
1897 	struct linger l;
1898 	struct timeval tv;
1899 
1900 	switch ((opt = sopt->sopt_name)) {
1901 
1902 	case SO_ACCEPTFILTER:
1903 		error = accept_filt_getopt(so, sopt);
1904 		break;
1905 
1906 	case SO_LINGER:
1907 		l.l_onoff = (so->so_options & SO_LINGER) ? 1 : 0;
1908 		l.l_linger = so->so_linger;
1909 
1910 		error = sockopt_set(sopt, &l, sizeof(l));
1911 		break;
1912 
1913 	case SO_USELOOPBACK:
1914 	case SO_DONTROUTE:
1915 	case SO_DEBUG:
1916 	case SO_KEEPALIVE:
1917 	case SO_REUSEADDR:
1918 	case SO_REUSEPORT:
1919 	case SO_BROADCAST:
1920 	case SO_OOBINLINE:
1921 	case SO_TIMESTAMP:
1922 #ifdef SO_OTIMESTAMP
1923 	case SO_OTIMESTAMP:
1924 #endif
1925 		error = sockopt_setint(sopt, (so->so_options & opt) ? 1 : 0);
1926 		break;
1927 
1928 	case SO_TYPE:
1929 		error = sockopt_setint(sopt, so->so_type);
1930 		break;
1931 
1932 	case SO_ERROR:
1933 		error = sockopt_setint(sopt, so->so_error);
1934 		so->so_error = 0;
1935 		break;
1936 
1937 	case SO_SNDBUF:
1938 		error = sockopt_setint(sopt, so->so_snd.sb_hiwat);
1939 		break;
1940 
1941 	case SO_RCVBUF:
1942 		error = sockopt_setint(sopt, so->so_rcv.sb_hiwat);
1943 		break;
1944 
1945 	case SO_SNDLOWAT:
1946 		error = sockopt_setint(sopt, so->so_snd.sb_lowat);
1947 		break;
1948 
1949 	case SO_RCVLOWAT:
1950 		error = sockopt_setint(sopt, so->so_rcv.sb_lowat);
1951 		break;
1952 
1953 #ifdef COMPAT_50
1954 	case SO_OSNDTIMEO:
1955 	case SO_ORCVTIMEO: {
1956 		struct timeval50 otv;
1957 
1958 		optval = (opt == SO_OSNDTIMEO ?
1959 		     so->so_snd.sb_timeo : so->so_rcv.sb_timeo);
1960 
1961 		otv.tv_sec = optval / hz;
1962 		otv.tv_usec = (optval % hz) * tick;
1963 
1964 		error = sockopt_set(sopt, &otv, sizeof(otv));
1965 		break;
1966 	}
1967 #endif /* COMPAT_50 */
1968 
1969 	case SO_SNDTIMEO:
1970 	case SO_RCVTIMEO:
1971 		optval = (opt == SO_SNDTIMEO ?
1972 		     so->so_snd.sb_timeo : so->so_rcv.sb_timeo);
1973 
1974 		tv.tv_sec = optval / hz;
1975 		tv.tv_usec = (optval % hz) * tick;
1976 
1977 		error = sockopt_set(sopt, &tv, sizeof(tv));
1978 		break;
1979 
1980 	case SO_OVERFLOWED:
1981 		error = sockopt_setint(sopt, so->so_rcv.sb_overflowed);
1982 		break;
1983 
1984 	default:
1985 		error = ENOPROTOOPT;
1986 		break;
1987 	}
1988 
1989 	return (error);
1990 }
1991 
1992 int
1993 sogetopt(struct socket *so, struct sockopt *sopt)
1994 {
1995 	int		error;
1996 
1997 	solock(so);
1998 	if (sopt->sopt_level != SOL_SOCKET) {
1999 		if (so->so_proto && so->so_proto->pr_ctloutput) {
2000 			error = ((*so->so_proto->pr_ctloutput)
2001 			    (PRCO_GETOPT, so, sopt));
2002 		} else
2003 			error = (ENOPROTOOPT);
2004 	} else {
2005 		error = sogetopt1(so, sopt);
2006 	}
2007 	sounlock(so);
2008 	return (error);
2009 }
2010 
2011 /*
2012  * alloc sockopt data buffer buffer
2013  *	- will be released at destroy
2014  */
2015 static int
2016 sockopt_alloc(struct sockopt *sopt, size_t len, km_flag_t kmflag)
2017 {
2018 
2019 	KASSERT(sopt->sopt_size == 0);
2020 
2021 	if (len > sizeof(sopt->sopt_buf)) {
2022 		sopt->sopt_data = kmem_zalloc(len, kmflag);
2023 		if (sopt->sopt_data == NULL)
2024 			return ENOMEM;
2025 	} else
2026 		sopt->sopt_data = sopt->sopt_buf;
2027 
2028 	sopt->sopt_size = len;
2029 	return 0;
2030 }
2031 
2032 /*
2033  * initialise sockopt storage
2034  *	- MAY sleep during allocation
2035  */
2036 void
2037 sockopt_init(struct sockopt *sopt, int level, int name, size_t size)
2038 {
2039 
2040 	memset(sopt, 0, sizeof(*sopt));
2041 
2042 	sopt->sopt_level = level;
2043 	sopt->sopt_name = name;
2044 	(void)sockopt_alloc(sopt, size, KM_SLEEP);
2045 }
2046 
2047 /*
2048  * destroy sockopt storage
2049  *	- will release any held memory references
2050  */
2051 void
2052 sockopt_destroy(struct sockopt *sopt)
2053 {
2054 
2055 	if (sopt->sopt_data != sopt->sopt_buf)
2056 		kmem_free(sopt->sopt_data, sopt->sopt_size);
2057 
2058 	memset(sopt, 0, sizeof(*sopt));
2059 }
2060 
2061 /*
2062  * set sockopt value
2063  *	- value is copied into sockopt
2064  * 	- memory is allocated when necessary, will not sleep
2065  */
2066 int
2067 sockopt_set(struct sockopt *sopt, const void *buf, size_t len)
2068 {
2069 	int error;
2070 
2071 	if (sopt->sopt_size == 0) {
2072 		error = sockopt_alloc(sopt, len, KM_NOSLEEP);
2073 		if (error)
2074 			return error;
2075 	}
2076 
2077 	KASSERT(sopt->sopt_size == len);
2078 	memcpy(sopt->sopt_data, buf, len);
2079 	return 0;
2080 }
2081 
2082 /*
2083  * common case of set sockopt integer value
2084  */
2085 int
2086 sockopt_setint(struct sockopt *sopt, int val)
2087 {
2088 
2089 	return sockopt_set(sopt, &val, sizeof(int));
2090 }
2091 
2092 /*
2093  * get sockopt value
2094  *	- correct size must be given
2095  */
2096 int
2097 sockopt_get(const struct sockopt *sopt, void *buf, size_t len)
2098 {
2099 
2100 	if (sopt->sopt_size != len)
2101 		return EINVAL;
2102 
2103 	memcpy(buf, sopt->sopt_data, len);
2104 	return 0;
2105 }
2106 
2107 /*
2108  * common case of get sockopt integer value
2109  */
2110 int
2111 sockopt_getint(const struct sockopt *sopt, int *valp)
2112 {
2113 
2114 	return sockopt_get(sopt, valp, sizeof(int));
2115 }
2116 
2117 /*
2118  * set sockopt value from mbuf
2119  *	- ONLY for legacy code
2120  *	- mbuf is released by sockopt
2121  *	- will not sleep
2122  */
2123 int
2124 sockopt_setmbuf(struct sockopt *sopt, struct mbuf *m)
2125 {
2126 	size_t len;
2127 	int error;
2128 
2129 	len = m_length(m);
2130 
2131 	if (sopt->sopt_size == 0) {
2132 		error = sockopt_alloc(sopt, len, KM_NOSLEEP);
2133 		if (error)
2134 			return error;
2135 	}
2136 
2137 	KASSERT(sopt->sopt_size == len);
2138 	m_copydata(m, 0, len, sopt->sopt_data);
2139 	m_freem(m);
2140 
2141 	return 0;
2142 }
2143 
2144 /*
2145  * get sockopt value into mbuf
2146  *	- ONLY for legacy code
2147  *	- mbuf to be released by the caller
2148  *	- will not sleep
2149  */
2150 struct mbuf *
2151 sockopt_getmbuf(const struct sockopt *sopt)
2152 {
2153 	struct mbuf *m;
2154 
2155 	if (sopt->sopt_size > MCLBYTES)
2156 		return NULL;
2157 
2158 	m = m_get(M_DONTWAIT, MT_SOOPTS);
2159 	if (m == NULL)
2160 		return NULL;
2161 
2162 	if (sopt->sopt_size > MLEN) {
2163 		MCLGET(m, M_DONTWAIT);
2164 		if ((m->m_flags & M_EXT) == 0) {
2165 			m_free(m);
2166 			return NULL;
2167 		}
2168 	}
2169 
2170 	memcpy(mtod(m, void *), sopt->sopt_data, sopt->sopt_size);
2171 	m->m_len = sopt->sopt_size;
2172 
2173 	return m;
2174 }
2175 
2176 void
2177 sohasoutofband(struct socket *so)
2178 {
2179 
2180 	fownsignal(so->so_pgid, SIGURG, POLL_PRI, POLLPRI|POLLRDBAND, so);
2181 	selnotify(&so->so_rcv.sb_sel, POLLPRI | POLLRDBAND, NOTE_SUBMIT);
2182 }
2183 
2184 static void
2185 filt_sordetach(struct knote *kn)
2186 {
2187 	struct socket	*so;
2188 
2189 	so = ((file_t *)kn->kn_obj)->f_data;
2190 	solock(so);
2191 	SLIST_REMOVE(&so->so_rcv.sb_sel.sel_klist, kn, knote, kn_selnext);
2192 	if (SLIST_EMPTY(&so->so_rcv.sb_sel.sel_klist))
2193 		so->so_rcv.sb_flags &= ~SB_KNOTE;
2194 	sounlock(so);
2195 }
2196 
2197 /*ARGSUSED*/
2198 static int
2199 filt_soread(struct knote *kn, long hint)
2200 {
2201 	struct socket	*so;
2202 	int rv;
2203 
2204 	so = ((file_t *)kn->kn_obj)->f_data;
2205 	if (hint != NOTE_SUBMIT)
2206 		solock(so);
2207 	kn->kn_data = so->so_rcv.sb_cc;
2208 	if (so->so_state & SS_CANTRCVMORE) {
2209 		kn->kn_flags |= EV_EOF;
2210 		kn->kn_fflags = so->so_error;
2211 		rv = 1;
2212 	} else if (so->so_error)	/* temporary udp error */
2213 		rv = 1;
2214 	else if (kn->kn_sfflags & NOTE_LOWAT)
2215 		rv = (kn->kn_data >= kn->kn_sdata);
2216 	else
2217 		rv = (kn->kn_data >= so->so_rcv.sb_lowat);
2218 	if (hint != NOTE_SUBMIT)
2219 		sounlock(so);
2220 	return rv;
2221 }
2222 
2223 static void
2224 filt_sowdetach(struct knote *kn)
2225 {
2226 	struct socket	*so;
2227 
2228 	so = ((file_t *)kn->kn_obj)->f_data;
2229 	solock(so);
2230 	SLIST_REMOVE(&so->so_snd.sb_sel.sel_klist, kn, knote, kn_selnext);
2231 	if (SLIST_EMPTY(&so->so_snd.sb_sel.sel_klist))
2232 		so->so_snd.sb_flags &= ~SB_KNOTE;
2233 	sounlock(so);
2234 }
2235 
2236 /*ARGSUSED*/
2237 static int
2238 filt_sowrite(struct knote *kn, long hint)
2239 {
2240 	struct socket	*so;
2241 	int rv;
2242 
2243 	so = ((file_t *)kn->kn_obj)->f_data;
2244 	if (hint != NOTE_SUBMIT)
2245 		solock(so);
2246 	kn->kn_data = sbspace(&so->so_snd);
2247 	if (so->so_state & SS_CANTSENDMORE) {
2248 		kn->kn_flags |= EV_EOF;
2249 		kn->kn_fflags = so->so_error;
2250 		rv = 1;
2251 	} else if (so->so_error)	/* temporary udp error */
2252 		rv = 1;
2253 	else if (((so->so_state & SS_ISCONNECTED) == 0) &&
2254 	    (so->so_proto->pr_flags & PR_CONNREQUIRED))
2255 		rv = 0;
2256 	else if (kn->kn_sfflags & NOTE_LOWAT)
2257 		rv = (kn->kn_data >= kn->kn_sdata);
2258 	else
2259 		rv = (kn->kn_data >= so->so_snd.sb_lowat);
2260 	if (hint != NOTE_SUBMIT)
2261 		sounlock(so);
2262 	return rv;
2263 }
2264 
2265 /*ARGSUSED*/
2266 static int
2267 filt_solisten(struct knote *kn, long hint)
2268 {
2269 	struct socket	*so;
2270 	int rv;
2271 
2272 	so = ((file_t *)kn->kn_obj)->f_data;
2273 
2274 	/*
2275 	 * Set kn_data to number of incoming connections, not
2276 	 * counting partial (incomplete) connections.
2277 	 */
2278 	if (hint != NOTE_SUBMIT)
2279 		solock(so);
2280 	kn->kn_data = so->so_qlen;
2281 	rv = (kn->kn_data > 0);
2282 	if (hint != NOTE_SUBMIT)
2283 		sounlock(so);
2284 	return rv;
2285 }
2286 
2287 static const struct filterops solisten_filtops =
2288 	{ 1, NULL, filt_sordetach, filt_solisten };
2289 static const struct filterops soread_filtops =
2290 	{ 1, NULL, filt_sordetach, filt_soread };
2291 static const struct filterops sowrite_filtops =
2292 	{ 1, NULL, filt_sowdetach, filt_sowrite };
2293 
2294 int
2295 soo_kqfilter(struct file *fp, struct knote *kn)
2296 {
2297 	struct socket	*so;
2298 	struct sockbuf	*sb;
2299 
2300 	so = ((file_t *)kn->kn_obj)->f_data;
2301 	solock(so);
2302 	switch (kn->kn_filter) {
2303 	case EVFILT_READ:
2304 		if (so->so_options & SO_ACCEPTCONN)
2305 			kn->kn_fop = &solisten_filtops;
2306 		else
2307 			kn->kn_fop = &soread_filtops;
2308 		sb = &so->so_rcv;
2309 		break;
2310 	case EVFILT_WRITE:
2311 		kn->kn_fop = &sowrite_filtops;
2312 		sb = &so->so_snd;
2313 		break;
2314 	default:
2315 		sounlock(so);
2316 		return (EINVAL);
2317 	}
2318 	SLIST_INSERT_HEAD(&sb->sb_sel.sel_klist, kn, kn_selnext);
2319 	sb->sb_flags |= SB_KNOTE;
2320 	sounlock(so);
2321 	return (0);
2322 }
2323 
2324 static int
2325 sodopoll(struct socket *so, int events)
2326 {
2327 	int revents;
2328 
2329 	revents = 0;
2330 
2331 	if (events & (POLLIN | POLLRDNORM))
2332 		if (soreadable(so))
2333 			revents |= events & (POLLIN | POLLRDNORM);
2334 
2335 	if (events & (POLLOUT | POLLWRNORM))
2336 		if (sowritable(so))
2337 			revents |= events & (POLLOUT | POLLWRNORM);
2338 
2339 	if (events & (POLLPRI | POLLRDBAND))
2340 		if (so->so_oobmark || (so->so_state & SS_RCVATMARK))
2341 			revents |= events & (POLLPRI | POLLRDBAND);
2342 
2343 	return revents;
2344 }
2345 
2346 int
2347 sopoll(struct socket *so, int events)
2348 {
2349 	int revents = 0;
2350 
2351 #ifndef DIAGNOSTIC
2352 	/*
2353 	 * Do a quick, unlocked check in expectation that the socket
2354 	 * will be ready for I/O.  Don't do this check if DIAGNOSTIC,
2355 	 * as the solocked() assertions will fail.
2356 	 */
2357 	if ((revents = sodopoll(so, events)) != 0)
2358 		return revents;
2359 #endif
2360 
2361 	solock(so);
2362 	if ((revents = sodopoll(so, events)) == 0) {
2363 		if (events & (POLLIN | POLLPRI | POLLRDNORM | POLLRDBAND)) {
2364 			selrecord(curlwp, &so->so_rcv.sb_sel);
2365 			so->so_rcv.sb_flags |= SB_NOTIFY;
2366 		}
2367 
2368 		if (events & (POLLOUT | POLLWRNORM)) {
2369 			selrecord(curlwp, &so->so_snd.sb_sel);
2370 			so->so_snd.sb_flags |= SB_NOTIFY;
2371 		}
2372 	}
2373 	sounlock(so);
2374 
2375 	return revents;
2376 }
2377 
2378 
2379 #include <sys/sysctl.h>
2380 
2381 static int sysctl_kern_somaxkva(SYSCTLFN_PROTO);
2382 
2383 /*
2384  * sysctl helper routine for kern.somaxkva.  ensures that the given
2385  * value is not too small.
2386  * (XXX should we maybe make sure it's not too large as well?)
2387  */
2388 static int
2389 sysctl_kern_somaxkva(SYSCTLFN_ARGS)
2390 {
2391 	int error, new_somaxkva;
2392 	struct sysctlnode node;
2393 
2394 	new_somaxkva = somaxkva;
2395 	node = *rnode;
2396 	node.sysctl_data = &new_somaxkva;
2397 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
2398 	if (error || newp == NULL)
2399 		return (error);
2400 
2401 	if (new_somaxkva < (16 * 1024 * 1024)) /* sanity */
2402 		return (EINVAL);
2403 
2404 	mutex_enter(&so_pendfree_lock);
2405 	somaxkva = new_somaxkva;
2406 	cv_broadcast(&socurkva_cv);
2407 	mutex_exit(&so_pendfree_lock);
2408 
2409 	return (error);
2410 }
2411 
2412 static void
2413 sysctl_kern_somaxkva_setup(void)
2414 {
2415 
2416 	KASSERT(socket_sysctllog == NULL);
2417 	sysctl_createv(&socket_sysctllog, 0, NULL, NULL,
2418 		       CTLFLAG_PERMANENT,
2419 		       CTLTYPE_NODE, "kern", NULL,
2420 		       NULL, 0, NULL, 0,
2421 		       CTL_KERN, CTL_EOL);
2422 
2423 	sysctl_createv(&socket_sysctllog, 0, NULL, NULL,
2424 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
2425 		       CTLTYPE_INT, "somaxkva",
2426 		       SYSCTL_DESCR("Maximum amount of kernel memory to be "
2427 				    "used for socket buffers"),
2428 		       sysctl_kern_somaxkva, 0, NULL, 0,
2429 		       CTL_KERN, KERN_SOMAXKVA, CTL_EOL);
2430 }
2431