1 /* 2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved. 3 * Copyright (c) 2004 The DragonFly Project. All rights reserved. 4 * 5 * This code is derived from software contributed to The DragonFly Project 6 * by Jeffrey M. Hsu. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of The DragonFly Project nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific, prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 1982, 1986, 1991, 1993, 1995 36 * The Regents of the University of California. All rights reserved. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. Neither the name of the University nor the names of its contributors 47 * may be used to endorse or promote products derived from this software 48 * without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 60 * SUCH DAMAGE. 61 * 62 * @(#)in_pcb.c 8.4 (Berkeley) 5/24/95 63 * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $ 64 */ 65 66 #include "opt_ipsec.h" 67 #include "opt_inet6.h" 68 69 #include <sys/param.h> 70 #include <sys/systm.h> 71 #include <sys/malloc.h> 72 #include <sys/mbuf.h> 73 #include <sys/domain.h> 74 #include <sys/protosw.h> 75 #include <sys/socket.h> 76 #include <sys/socketvar.h> 77 #include <sys/proc.h> 78 #include <sys/priv.h> 79 #include <sys/jail.h> 80 #include <sys/kernel.h> 81 #include <sys/sysctl.h> 82 83 #include <sys/thread2.h> 84 #include <sys/socketvar2.h> 85 #include <sys/msgport2.h> 86 87 #include <machine/limits.h> 88 89 #include <net/if.h> 90 #include <net/if_types.h> 91 #include <net/route.h> 92 #include <net/netisr2.h> 93 #include <net/toeplitz2.h> 94 95 #include <netinet/in.h> 96 #include <netinet/in_pcb.h> 97 #include <netinet/in_var.h> 98 #include <netinet/ip_var.h> 99 #ifdef INET6 100 #include <netinet/ip6.h> 101 #include <netinet6/ip6_var.h> 102 #endif /* INET6 */ 103 104 #ifdef IPSEC 105 #include <netinet6/ipsec.h> 106 #include <netproto/key/key.h> 107 #include <netproto/ipsec/esp_var.h> 108 #endif 109 110 #ifdef FAST_IPSEC 111 #if defined(IPSEC) || defined(IPSEC_ESP) 112 #error "Bad idea: don't compile with both IPSEC and FAST_IPSEC!" 113 #endif 114 115 #include <netproto/ipsec/ipsec.h> 116 #include <netproto/ipsec/key.h> 117 #define IPSEC 118 #endif /* FAST_IPSEC */ 119 120 #define INP_LOCALGROUP_SIZMIN 8 121 #define INP_LOCALGROUP_SIZMAX 256 122 123 struct in_addr zeroin_addr; 124 125 /* 126 * These configure the range of local port addresses assigned to 127 * "unspecified" outgoing connections/packets/whatever. 128 */ 129 int ipport_lowfirstauto = IPPORT_RESERVED - 1; /* 1023 */ 130 int ipport_lowlastauto = IPPORT_RESERVEDSTART; /* 600 */ 131 132 int ipport_firstauto = IPPORT_RESERVED; /* 1024 */ 133 int ipport_lastauto = IPPORT_USERRESERVED; /* 5000 */ 134 135 int ipport_hifirstauto = IPPORT_HIFIRSTAUTO; /* 49152 */ 136 int ipport_hilastauto = IPPORT_HILASTAUTO; /* 65535 */ 137 138 #define RANGECHK(var, min, max) \ 139 if ((var) < (min)) { (var) = (min); } \ 140 else if ((var) > (max)) { (var) = (max); } 141 142 int udpencap_enable = 1; /* enabled by default */ 143 int udpencap_port = 4500; /* triggers decapsulation */ 144 145 /* 146 * Per-netisr inpcb markers. 147 * NOTE: they should only be used in netisrs. 148 */ 149 static struct inpcb *in_pcbmarkers; 150 static struct inpcontainer *in_pcbcontainer_markers; 151 152 static int 153 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS) 154 { 155 int error; 156 157 error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req); 158 if (!error) { 159 RANGECHK(ipport_lowfirstauto, 1, IPPORT_RESERVED - 1); 160 RANGECHK(ipport_lowlastauto, 1, IPPORT_RESERVED - 1); 161 162 RANGECHK(ipport_firstauto, IPPORT_RESERVED, USHRT_MAX); 163 RANGECHK(ipport_lastauto, IPPORT_RESERVED, USHRT_MAX); 164 165 RANGECHK(ipport_hifirstauto, IPPORT_RESERVED, USHRT_MAX); 166 RANGECHK(ipport_hilastauto, IPPORT_RESERVED, USHRT_MAX); 167 } 168 return (error); 169 } 170 171 #undef RANGECHK 172 173 SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports"); 174 175 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW, 176 &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", ""); 177 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, CTLTYPE_INT|CTLFLAG_RW, 178 &ipport_lowlastauto, 0, &sysctl_net_ipport_check, "I", ""); 179 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, CTLTYPE_INT|CTLFLAG_RW, 180 &ipport_firstauto, 0, &sysctl_net_ipport_check, "I", ""); 181 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, CTLTYPE_INT|CTLFLAG_RW, 182 &ipport_lastauto, 0, &sysctl_net_ipport_check, "I", ""); 183 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, CTLTYPE_INT|CTLFLAG_RW, 184 &ipport_hifirstauto, 0, &sysctl_net_ipport_check, "I", ""); 185 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW, 186 &ipport_hilastauto, 0, &sysctl_net_ipport_check, "I", ""); 187 188 static int ip_porthash_trycount = 15; 189 SYSCTL_INT(_net_inet_ip, OID_AUTO, porthash_trycount, CTLFLAG_RW, 190 &ip_porthash_trycount, 0, 191 "Number of tries to find local port matching hash of 4-tuple"); 192 193 /* 194 * in_pcb.c: manage the Protocol Control Blocks. 195 * 196 * NOTE: It is assumed that most of these functions will be called from 197 * a critical section. XXX - There are, unfortunately, a few exceptions 198 * to this rule that should be fixed. 199 * 200 * NOTE: The caller should initialize the cpu field to the cpu running the 201 * protocol stack associated with this inpcbinfo. 202 */ 203 204 void 205 in_pcbinfo_init(struct inpcbinfo *pcbinfo, int cpu, boolean_t shared) 206 { 207 KASSERT(cpu >= 0 && cpu < ncpus, ("invalid cpu%d", cpu)); 208 pcbinfo->cpu = cpu; 209 210 LIST_INIT(&pcbinfo->pcblisthead); 211 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), M_PCB, 212 M_WAITOK | M_ZERO); 213 214 if (shared) { 215 pcbinfo->infotoken = kmalloc(sizeof(struct lwkt_token), 216 M_PCB, M_WAITOK); 217 lwkt_token_init(pcbinfo->infotoken, "infotoken"); 218 } else { 219 pcbinfo->infotoken = NULL; 220 } 221 } 222 223 struct baddynamicports baddynamicports; 224 225 /* 226 * Check if the specified port is invalid for dynamic allocation. 227 */ 228 int 229 in_baddynamic(u_int16_t port, u_int16_t proto) 230 { 231 switch (proto) { 232 case IPPROTO_TCP: 233 return (DP_ISSET(baddynamicports.tcp, port)); 234 case IPPROTO_UDP: 235 #ifdef IPSEC 236 /* Cannot preset this as it is a sysctl */ 237 if (port == udpencap_port) 238 return (1); 239 #endif 240 return (DP_ISSET(baddynamicports.udp, port)); 241 default: 242 return (0); 243 } 244 } 245 246 void 247 in_pcbonlist(struct inpcb *inp) 248 { 249 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 250 251 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 252 ("not in the correct netisr")); 253 KASSERT((inp->inp_flags & INP_ONLIST) == 0, ("already on pcblist")); 254 inp->inp_flags |= INP_ONLIST; 255 256 GET_PCBINFO_TOKEN(pcbinfo); 257 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list); 258 pcbinfo->ipi_count++; 259 REL_PCBINFO_TOKEN(pcbinfo); 260 } 261 262 void 263 in_pcbofflist(struct inpcb *inp) 264 { 265 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 266 267 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 268 ("not in the correct netisr")); 269 KASSERT(inp->inp_flags & INP_ONLIST, ("not on pcblist")); 270 inp->inp_flags &= ~INP_ONLIST; 271 272 GET_PCBINFO_TOKEN(pcbinfo); 273 LIST_REMOVE(inp, inp_list); 274 KASSERT(pcbinfo->ipi_count > 0, 275 ("invalid inpcb count %d", pcbinfo->ipi_count)); 276 pcbinfo->ipi_count--; 277 REL_PCBINFO_TOKEN(pcbinfo); 278 } 279 280 /* 281 * Allocate a PCB and associate it with the socket. 282 */ 283 int 284 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo) 285 { 286 struct inpcb *inp; 287 #ifdef IPSEC 288 int error; 289 #endif 290 291 inp = kmalloc(pcbinfo->ipi_size, M_PCB, M_WAITOK|M_ZERO|M_NULLOK); 292 if (inp == NULL) 293 return (ENOMEM); 294 inp->inp_lgrpindex = -1; 295 inp->inp_gencnt = ++pcbinfo->ipi_gencnt; 296 inp->inp_pcbinfo = pcbinfo; 297 inp->inp_socket = so; 298 #ifdef IPSEC 299 error = ipsec_init_policy(so, &inp->inp_sp); 300 if (error != 0) { 301 kfree(inp, M_PCB); 302 return (error); 303 } 304 #endif 305 #ifdef INET6 306 if (INP_CHECK_SOCKAF(so, AF_INET6)) { 307 if (ip6_auto_flowlabel) 308 inp->inp_flags |= IN6P_AUTOFLOWLABEL; 309 inp->inp_af = AF_INET6; 310 } else 311 #endif 312 inp->inp_af = AF_INET; 313 soreference(so); 314 so->so_pcb = inp; 315 316 in_pcbonlist(inp); 317 return (0); 318 } 319 320 /* 321 * Unlink a pcb with the intention of moving it to another cpu with a 322 * different pcbinfo. While unlinked nothing should attempt to dereference 323 * inp_pcbinfo, NULL it out so we assert if it does. 324 */ 325 void 326 in_pcbunlink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags) 327 { 328 KASSERT(inp->inp_pcbinfo == pcbinfo, ("pcbinfo mismatch")); 329 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0, 330 ("already linked")); 331 332 in_pcbofflist(inp); 333 inp->inp_pcbinfo = NULL; 334 } 335 336 void 337 in_pcbunlink(struct inpcb *inp, struct inpcbinfo *pcbinfo) 338 { 339 in_pcbunlink_flags(inp, pcbinfo, INP_WILDCARD); 340 } 341 342 /* 343 * Relink a pcb into a new pcbinfo. 344 */ 345 void 346 in_pcblink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags) 347 { 348 KASSERT(inp->inp_pcbinfo == NULL, ("has pcbinfo")); 349 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0, 350 ("already linked")); 351 352 inp->inp_pcbinfo = pcbinfo; 353 in_pcbonlist(inp); 354 } 355 356 void 357 in_pcblink(struct inpcb *inp, struct inpcbinfo *pcbinfo) 358 { 359 return in_pcblink_flags(inp, pcbinfo, INP_WILDCARD); 360 } 361 362 static int 363 in_pcbsetlport(struct inpcb *inp, int wild, struct ucred *cred) 364 { 365 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 366 struct inpcbportinfo *portinfo; 367 u_short first, last, lport, step; 368 u_short *lastport; 369 int count, error; 370 int portinfo_first, portinfo_idx; 371 372 inp->inp_flags |= INP_ANONPORT; 373 374 step = pcbinfo->portinfo_mask + 1; 375 portinfo_first = mycpuid & pcbinfo->portinfo_mask; 376 portinfo_idx = portinfo_first; 377 loop: 378 portinfo = &pcbinfo->portinfo[portinfo_idx]; 379 380 if (inp->inp_flags & INP_HIGHPORT) { 381 first = ipport_hifirstauto; /* sysctl */ 382 last = ipport_hilastauto; 383 lastport = &portinfo->lasthi; 384 } else if (inp->inp_flags & INP_LOWPORT) { 385 if (cred && 386 (error = 387 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 388 inp->inp_laddr.s_addr = INADDR_ANY; 389 return error; 390 } 391 first = ipport_lowfirstauto; /* 1023 */ 392 last = ipport_lowlastauto; /* 600 */ 393 lastport = &portinfo->lastlow; 394 } else { 395 first = ipport_firstauto; /* sysctl */ 396 last = ipport_lastauto; 397 lastport = &portinfo->lastport; 398 } 399 400 /* 401 * This has to be atomic. If the porthash is shared across multiple 402 * protocol threads (aka tcp) then the token must be held. 403 */ 404 GET_PORT_TOKEN(portinfo); 405 406 /* 407 * Simple check to ensure all ports are not used up causing 408 * a deadlock here. 409 * 410 * We split the two cases (up and down) so that the direction 411 * is not being tested on each round of the loop. 412 */ 413 if (first > last) { 414 /* 415 * counting down 416 */ 417 in_pcbportrange(&first, &last, portinfo->offset, step); 418 count = (first - last) / step; 419 420 do { 421 if (count-- < 0) { /* completely used? */ 422 error = EADDRNOTAVAIL; 423 goto done; 424 } 425 *lastport -= step; 426 if (*lastport > first || *lastport < last) 427 *lastport = first; 428 KKASSERT((*lastport & pcbinfo->portinfo_mask) == 429 portinfo->offset); 430 lport = htons(*lastport); 431 } while (in_pcblookup_local(portinfo, inp->inp_laddr, lport, 432 wild, cred)); 433 } else { 434 /* 435 * counting up 436 */ 437 in_pcbportrange(&last, &first, portinfo->offset, step); 438 count = (last - first) / step; 439 440 do { 441 if (count-- < 0) { /* completely used? */ 442 error = EADDRNOTAVAIL; 443 goto done; 444 } 445 *lastport += step; 446 if (*lastport < first || *lastport > last) 447 *lastport = first; 448 KKASSERT((*lastport & pcbinfo->portinfo_mask) == 449 portinfo->offset); 450 lport = htons(*lastport); 451 } while (in_pcblookup_local(portinfo, inp->inp_laddr, lport, 452 wild, cred)); 453 } 454 inp->inp_lport = lport; 455 in_pcbinsporthash(portinfo, inp); 456 error = 0; 457 done: 458 REL_PORT_TOKEN(portinfo); 459 460 if (error) { 461 /* Try next portinfo */ 462 portinfo_idx++; 463 portinfo_idx &= pcbinfo->portinfo_mask; 464 if (portinfo_idx != portinfo_first) 465 goto loop; 466 inp->inp_laddr.s_addr = INADDR_ANY; 467 } 468 return error; 469 } 470 471 int 472 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct thread *td) 473 { 474 struct socket *so = inp->inp_socket; 475 struct sockaddr_in jsin; 476 struct ucred *cred = NULL; 477 int wild = 0; 478 479 if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */ 480 return (EADDRNOTAVAIL); 481 if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY) 482 return (EINVAL); /* already bound */ 483 484 if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT))) 485 wild = 1; /* neither SO_REUSEADDR nor SO_REUSEPORT is set */ 486 if (td->td_proc) 487 cred = td->td_proc->p_ucred; 488 489 if (nam != NULL) { 490 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 491 struct inpcbinfo *pcbinfo; 492 struct inpcbportinfo *portinfo; 493 struct inpcb *t; 494 u_short lport, lport_ho; 495 int reuseport = (so->so_options & SO_REUSEPORT); 496 int error; 497 498 if (nam->sa_len != sizeof *sin) 499 return (EINVAL); 500 #ifdef notdef 501 /* 502 * We should check the family, but old programs 503 * incorrectly fail to initialize it. 504 */ 505 if (sin->sin_family != AF_INET) 506 return (EAFNOSUPPORT); 507 #endif 508 if (!prison_replace_wildcards(td, nam)) 509 return (EINVAL); 510 511 lport = sin->sin_port; 512 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { 513 /* 514 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast; 515 * allow complete duplication of binding if 516 * SO_REUSEPORT is set, or if SO_REUSEADDR is set 517 * and a multicast address is bound on both 518 * new and duplicated sockets. 519 */ 520 if (so->so_options & SO_REUSEADDR) 521 reuseport = SO_REUSEADDR | SO_REUSEPORT; 522 } else if (sin->sin_addr.s_addr != INADDR_ANY) { 523 sin->sin_port = 0; /* yech... */ 524 bzero(&sin->sin_zero, sizeof sin->sin_zero); 525 if (ifa_ifwithaddr((struct sockaddr *)sin) == NULL) 526 return (EADDRNOTAVAIL); 527 } 528 529 inp->inp_laddr = sin->sin_addr; 530 531 jsin.sin_family = AF_INET; 532 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 533 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 534 inp->inp_laddr.s_addr = INADDR_ANY; 535 return (EINVAL); 536 } 537 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 538 539 if (lport == 0) { 540 /* Auto-select local port */ 541 return in_pcbsetlport(inp, wild, cred); 542 } 543 lport_ho = ntohs(lport); 544 545 /* GROSS */ 546 if (lport_ho < IPPORT_RESERVED && cred && 547 (error = 548 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 549 inp->inp_laddr.s_addr = INADDR_ANY; 550 return (error); 551 } 552 553 /* 554 * Locate the proper portinfo based on lport 555 */ 556 pcbinfo = inp->inp_pcbinfo; 557 portinfo = 558 &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask]; 559 KKASSERT((lport_ho & pcbinfo->portinfo_mask) == 560 portinfo->offset); 561 562 /* 563 * This has to be atomic. If the porthash is shared across 564 * multiple protocol threads (aka tcp) then the token must 565 * be held. 566 */ 567 GET_PORT_TOKEN(portinfo); 568 569 if (so->so_cred->cr_uid != 0 && 570 !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) { 571 t = in_pcblookup_local(portinfo, sin->sin_addr, lport, 572 INPLOOKUP_WILDCARD, cred); 573 if (t && 574 (so->so_cred->cr_uid != 575 t->inp_socket->so_cred->cr_uid)) { 576 inp->inp_laddr.s_addr = INADDR_ANY; 577 error = EADDRINUSE; 578 goto done; 579 } 580 } 581 if (cred && !prison_replace_wildcards(td, nam)) { 582 inp->inp_laddr.s_addr = INADDR_ANY; 583 error = EADDRNOTAVAIL; 584 goto done; 585 } 586 t = in_pcblookup_local(portinfo, sin->sin_addr, lport, 587 wild, cred); 588 if (t && !(reuseport & t->inp_socket->so_options)) { 589 inp->inp_laddr.s_addr = INADDR_ANY; 590 error = EADDRINUSE; 591 goto done; 592 } 593 inp->inp_lport = lport; 594 in_pcbinsporthash(portinfo, inp); 595 error = 0; 596 done: 597 REL_PORT_TOKEN(portinfo); 598 return (error); 599 } else { 600 jsin.sin_family = AF_INET; 601 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 602 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 603 inp->inp_laddr.s_addr = INADDR_ANY; 604 return (EINVAL); 605 } 606 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 607 608 return in_pcbsetlport(inp, wild, cred); 609 } 610 } 611 612 static struct inpcb * 613 in_pcblookup_localremote(struct inpcbportinfo *portinfo, struct in_addr laddr, 614 u_short lport, struct in_addr faddr, u_short fport, struct ucred *cred) 615 { 616 struct inpcb *inp; 617 struct inpcbporthead *porthash; 618 struct inpcbport *phd; 619 struct inpcb *match = NULL; 620 621 /* 622 * If the porthashbase is shared across several cpus, it must 623 * have been locked. 624 */ 625 ASSERT_PORT_TOKEN_HELD(portinfo); 626 627 /* 628 * Best fit PCB lookup. 629 * 630 * First see if this local port is in use by looking on the 631 * port hash list. 632 */ 633 porthash = &portinfo->porthashbase[ 634 INP_PCBPORTHASH(lport, portinfo->porthashmask)]; 635 LIST_FOREACH(phd, porthash, phd_hash) { 636 if (phd->phd_port == lport) 637 break; 638 } 639 if (phd != NULL) { 640 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 641 #ifdef INET6 642 if (!INP_ISIPV4(inp)) 643 continue; 644 #endif 645 if (inp->inp_laddr.s_addr != INADDR_ANY && 646 inp->inp_laddr.s_addr != laddr.s_addr) 647 continue; 648 649 if (inp->inp_faddr.s_addr != INADDR_ANY && 650 inp->inp_faddr.s_addr != faddr.s_addr) 651 continue; 652 653 if (inp->inp_fport != 0 && inp->inp_fport != fport) 654 continue; 655 656 if (cred == NULL || 657 cred->cr_prison == 658 inp->inp_socket->so_cred->cr_prison) { 659 match = inp; 660 break; 661 } 662 } 663 } 664 return (match); 665 } 666 667 int 668 in_pcbbind_remote(struct inpcb *inp, const struct sockaddr *remote, 669 struct thread *td) 670 { 671 struct proc *p = td->td_proc; 672 unsigned short *lastport; 673 const struct sockaddr_in *sin = (const struct sockaddr_in *)remote; 674 struct sockaddr_in jsin; 675 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 676 struct inpcbportinfo *portinfo; 677 struct ucred *cred = NULL; 678 u_short first, last, lport, step; 679 int count, error, dup; 680 int portinfo_first, portinfo_idx; 681 int hash_cpu, hash_count, hash_count0; 682 uint32_t hash_base = 0, hash; 683 684 if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */ 685 return (EADDRNOTAVAIL); 686 687 KKASSERT(inp->inp_laddr.s_addr != INADDR_ANY); 688 if (inp->inp_lport != 0) 689 return (EINVAL); /* already bound */ 690 691 KKASSERT(p); 692 cred = p->p_ucred; 693 694 jsin.sin_family = AF_INET; 695 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr; 696 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) { 697 inp->inp_laddr.s_addr = INADDR_ANY; 698 return (EINVAL); 699 } 700 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr; 701 702 hash_count0 = ip_porthash_trycount; 703 if (hash_count0 > 0) { 704 hash_base = toeplitz_piecemeal_addr(sin->sin_addr.s_addr) ^ 705 toeplitz_piecemeal_addr(inp->inp_laddr.s_addr) ^ 706 toeplitz_piecemeal_port(sin->sin_port); 707 } else { 708 hash_count0 = 0; 709 } 710 711 inp->inp_flags |= INP_ANONPORT; 712 713 step = pcbinfo->portinfo_mask + 1; 714 portinfo_first = mycpuid & pcbinfo->portinfo_mask; 715 portinfo_idx = portinfo_first; 716 loop: 717 hash_cpu = portinfo_idx & ncpus2_mask; 718 portinfo = &pcbinfo->portinfo[portinfo_idx]; 719 dup = 0; 720 721 if (inp->inp_flags & INP_HIGHPORT) { 722 first = ipport_hifirstauto; /* sysctl */ 723 last = ipport_hilastauto; 724 lastport = &portinfo->lasthi; 725 } else if (inp->inp_flags & INP_LOWPORT) { 726 if (cred && 727 (error = 728 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) { 729 inp->inp_laddr.s_addr = INADDR_ANY; 730 return (error); 731 } 732 first = ipport_lowfirstauto; /* 1023 */ 733 last = ipport_lowlastauto; /* 600 */ 734 lastport = &portinfo->lastlow; 735 } else { 736 first = ipport_firstauto; /* sysctl */ 737 last = ipport_lastauto; 738 lastport = &portinfo->lastport; 739 } 740 741 /* 742 * This has to be atomic. If the porthash is shared across multiple 743 * protocol threads (aka tcp) then the token must be held. 744 */ 745 GET_PORT_TOKEN(portinfo); 746 747 again: 748 /* 749 * Simple check to ensure all ports are not used up causing 750 * a deadlock here. 751 * 752 * We split the two cases (up and down) so that the direction 753 * is not being tested on each round of the loop. 754 */ 755 hash_count = hash_count0; 756 if (first > last) { 757 /* 758 * counting down 759 */ 760 in_pcbportrange(&first, &last, portinfo->offset, step); 761 count = ((first - last) / step) + hash_count; 762 763 for (;;) { 764 if (count-- < 0) { /* completely used? */ 765 error = EADDRNOTAVAIL; 766 goto done; 767 } 768 *lastport -= step; 769 if (*lastport > first || *lastport < last) 770 *lastport = first; 771 KKASSERT((*lastport & pcbinfo->portinfo_mask) == 772 portinfo->offset); 773 lport = htons(*lastport); 774 775 if (hash_count) { 776 --hash_count; 777 hash = hash_base ^ 778 toeplitz_piecemeal_port(lport); 779 if ((hash & ncpus2_mask) != hash_cpu && hash_count) 780 continue; 781 } 782 783 if (in_pcblookup_localremote(portinfo, inp->inp_laddr, 784 lport, sin->sin_addr, sin->sin_port, cred) == NULL) 785 break; 786 } 787 } else { 788 /* 789 * counting up 790 */ 791 in_pcbportrange(&last, &first, portinfo->offset, step); 792 count = ((last - first) / step) + hash_count; 793 794 for (;;) { 795 if (count-- < 0) { /* completely used? */ 796 error = EADDRNOTAVAIL; 797 goto done; 798 } 799 *lastport += step; 800 if (*lastport < first || *lastport > last) 801 *lastport = first; 802 KKASSERT((*lastport & pcbinfo->portinfo_mask) == 803 portinfo->offset); 804 lport = htons(*lastport); 805 806 if (hash_count) { 807 --hash_count; 808 hash = hash_base ^ 809 toeplitz_piecemeal_port(lport); 810 if ((hash & ncpus2_mask) != hash_cpu && hash_count) 811 continue; 812 } 813 814 if (in_pcblookup_localremote(portinfo, inp->inp_laddr, 815 lport, sin->sin_addr, sin->sin_port, cred) == NULL) 816 break; 817 } 818 } 819 820 /* This could happen on loopback interface */ 821 if (sin->sin_port == lport && 822 sin->sin_addr.s_addr == inp->inp_laddr.s_addr) { 823 if (dup) { 824 /* 825 * Duplicate again; give up 826 */ 827 error = EADDRNOTAVAIL; 828 goto done; 829 } 830 dup = 1; 831 goto again; 832 } 833 inp->inp_lport = lport; 834 in_pcbinsporthash(portinfo, inp); 835 error = 0; 836 done: 837 REL_PORT_TOKEN(portinfo); 838 839 if (error) { 840 /* Try next portinfo */ 841 portinfo_idx++; 842 portinfo_idx &= pcbinfo->portinfo_mask; 843 if (portinfo_idx != portinfo_first) 844 goto loop; 845 inp->inp_laddr.s_addr = INADDR_ANY; 846 } 847 return error; 848 } 849 850 /* 851 * Transform old in_pcbconnect() into an inner subroutine for new 852 * in_pcbconnect(): Do some validity-checking on the remote 853 * address (in mbuf 'nam') and then determine local host address 854 * (i.e., which interface) to use to access that remote host. 855 * 856 * This preserves definition of in_pcbconnect(), while supporting a 857 * slightly different version for T/TCP. (This is more than 858 * a bit of a kludge, but cleaning up the internal interfaces would 859 * have forced minor changes in every protocol). 860 */ 861 int 862 in_pcbladdr_find(struct inpcb *inp, struct sockaddr *nam, 863 struct sockaddr_in **plocal_sin, struct thread *td, int find) 864 { 865 struct in_ifaddr *ia; 866 struct ucred *cred = NULL; 867 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 868 struct sockaddr *jsin; 869 int jailed = 0, alloc_route = 0; 870 871 if (nam->sa_len != sizeof *sin) 872 return (EINVAL); 873 if (sin->sin_family != AF_INET) 874 return (EAFNOSUPPORT); 875 if (sin->sin_port == 0) 876 return (EADDRNOTAVAIL); 877 if (td && td->td_proc && td->td_proc->p_ucred) 878 cred = td->td_proc->p_ucred; 879 if (cred && cred->cr_prison) 880 jailed = 1; 881 if (!TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) { 882 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia; 883 /* 884 * If the destination address is INADDR_ANY, 885 * use the primary local address. 886 * If the supplied address is INADDR_BROADCAST, 887 * and the primary interface supports broadcast, 888 * choose the broadcast address for that interface. 889 */ 890 if (sin->sin_addr.s_addr == INADDR_ANY) 891 sin->sin_addr = IA_SIN(ia)->sin_addr; 892 else if (sin->sin_addr.s_addr == (u_long)INADDR_BROADCAST && 893 (ia->ia_ifp->if_flags & IFF_BROADCAST)) 894 sin->sin_addr = satosin(&ia->ia_broadaddr)->sin_addr; 895 } 896 if (find) { 897 struct route *ro; 898 899 ia = NULL; 900 /* 901 * If route is known or can be allocated now, 902 * our src addr is taken from the i/f, else punt. 903 * Note that we should check the address family of the cached 904 * destination, in case of sharing the cache with IPv6. 905 */ 906 ro = &inp->inp_route; 907 if (ro->ro_rt && 908 (!(ro->ro_rt->rt_flags & RTF_UP) || 909 ro->ro_dst.sa_family != AF_INET || 910 satosin(&ro->ro_dst)->sin_addr.s_addr != 911 sin->sin_addr.s_addr || 912 inp->inp_socket->so_options & SO_DONTROUTE)) { 913 RTFREE(ro->ro_rt); 914 ro->ro_rt = NULL; 915 } 916 if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/ 917 (ro->ro_rt == NULL || 918 ro->ro_rt->rt_ifp == NULL)) { 919 /* No route yet, so try to acquire one */ 920 bzero(&ro->ro_dst, sizeof(struct sockaddr_in)); 921 ro->ro_dst.sa_family = AF_INET; 922 ro->ro_dst.sa_len = sizeof(struct sockaddr_in); 923 ((struct sockaddr_in *) &ro->ro_dst)->sin_addr = 924 sin->sin_addr; 925 rtalloc(ro); 926 alloc_route = 1; 927 } 928 /* 929 * If we found a route, use the address 930 * corresponding to the outgoing interface 931 * unless it is the loopback (in case a route 932 * to our address on another net goes to loopback). 933 */ 934 if (ro->ro_rt && 935 !(ro->ro_rt->rt_ifp->if_flags & IFF_LOOPBACK)) { 936 if (jailed) { 937 if (jailed_ip(cred->cr_prison, 938 ro->ro_rt->rt_ifa->ifa_addr)) { 939 ia = ifatoia(ro->ro_rt->rt_ifa); 940 } 941 } else { 942 ia = ifatoia(ro->ro_rt->rt_ifa); 943 } 944 } 945 if (ia == NULL) { 946 u_short fport = sin->sin_port; 947 948 sin->sin_port = 0; 949 ia = ifatoia(ifa_ifwithdstaddr(sintosa(sin))); 950 if (ia && jailed && !jailed_ip(cred->cr_prison, 951 sintosa(&ia->ia_addr))) 952 ia = NULL; 953 if (ia == NULL) 954 ia = ifatoia(ifa_ifwithnet(sintosa(sin))); 955 if (ia && jailed && !jailed_ip(cred->cr_prison, 956 sintosa(&ia->ia_addr))) 957 ia = NULL; 958 sin->sin_port = fport; 959 if (ia == NULL && 960 !TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) 961 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia; 962 if (ia && jailed && !jailed_ip(cred->cr_prison, 963 sintosa(&ia->ia_addr))) 964 ia = NULL; 965 966 if (!jailed && ia == NULL) 967 goto fail; 968 } 969 /* 970 * If the destination address is multicast and an outgoing 971 * interface has been set as a multicast option, use the 972 * address of that interface as our source address. 973 */ 974 if (!jailed && IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) && 975 inp->inp_moptions != NULL) { 976 struct ip_moptions *imo; 977 struct ifnet *ifp; 978 979 imo = inp->inp_moptions; 980 if (imo->imo_multicast_ifp != NULL) { 981 struct in_ifaddr_container *iac; 982 983 ifp = imo->imo_multicast_ifp; 984 ia = NULL; 985 TAILQ_FOREACH(iac, 986 &in_ifaddrheads[mycpuid], ia_link) { 987 if (iac->ia->ia_ifp == ifp) { 988 ia = iac->ia; 989 break; 990 } 991 } 992 if (ia == NULL) 993 goto fail; 994 } 995 } 996 /* 997 * Don't do pcblookup call here; return interface in plocal_sin 998 * and exit to caller, that will do the lookup. 999 */ 1000 if (ia == NULL && jailed) { 1001 if ((jsin = prison_get_nonlocal( 1002 cred->cr_prison, AF_INET, NULL)) != NULL || 1003 (jsin = prison_get_local( 1004 cred->cr_prison, AF_INET, NULL)) != NULL) { 1005 *plocal_sin = satosin(jsin); 1006 } else { 1007 /* IPv6 only Jail */ 1008 goto fail; 1009 } 1010 } else { 1011 *plocal_sin = &ia->ia_addr; 1012 } 1013 } 1014 return (0); 1015 fail: 1016 if (alloc_route) 1017 in_pcbresetroute(inp); 1018 return (EADDRNOTAVAIL); 1019 } 1020 1021 int 1022 in_pcbladdr(struct inpcb *inp, struct sockaddr *nam, 1023 struct sockaddr_in **plocal_sin, struct thread *td) 1024 { 1025 return in_pcbladdr_find(inp, nam, plocal_sin, td, 1026 (inp->inp_laddr.s_addr == INADDR_ANY)); 1027 } 1028 1029 /* 1030 * Outer subroutine: 1031 * Connect from a socket to a specified address. 1032 * Both address and port must be specified in argument sin. 1033 * If don't have a local address for this socket yet, 1034 * then pick one. 1035 */ 1036 int 1037 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct thread *td) 1038 { 1039 struct sockaddr_in *if_sin; 1040 struct sockaddr_in *sin = (struct sockaddr_in *)nam; 1041 int error; 1042 1043 /* Call inner routine to assign local interface address. */ 1044 if ((error = in_pcbladdr(inp, nam, &if_sin, td)) != 0) 1045 return (error); 1046 1047 if (in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port, 1048 inp->inp_laddr.s_addr ? 1049 inp->inp_laddr : if_sin->sin_addr, 1050 inp->inp_lport, FALSE, NULL) != NULL) { 1051 return (EADDRINUSE); 1052 } 1053 if (inp->inp_laddr.s_addr == INADDR_ANY) { 1054 if (inp->inp_lport == 0) { 1055 error = in_pcbbind(inp, NULL, td); 1056 if (error) 1057 return (error); 1058 } 1059 inp->inp_laddr = if_sin->sin_addr; 1060 } 1061 inp->inp_faddr = sin->sin_addr; 1062 inp->inp_fport = sin->sin_port; 1063 in_pcbinsconnhash(inp); 1064 return (0); 1065 } 1066 1067 void 1068 in_pcbdisconnect(struct inpcb *inp) 1069 { 1070 1071 in_pcbremconnhash(inp); 1072 inp->inp_faddr.s_addr = INADDR_ANY; 1073 inp->inp_fport = 0; 1074 } 1075 1076 void 1077 in_pcbdetach(struct inpcb *inp) 1078 { 1079 struct socket *so = inp->inp_socket; 1080 struct inpcbinfo *ipi = inp->inp_pcbinfo; 1081 1082 #ifdef IPSEC 1083 ipsec4_delete_pcbpolicy(inp); 1084 #endif /*IPSEC*/ 1085 inp->inp_gencnt = ++ipi->ipi_gencnt; 1086 KKASSERT((so->so_state & SS_ASSERTINPROG) == 0); 1087 in_pcbremlists(inp); 1088 so->so_pcb = NULL; 1089 sofree(so); /* remove pcb ref */ 1090 if (inp->inp_options) 1091 m_free(inp->inp_options); 1092 if (inp->inp_route.ro_rt) 1093 rtfree(inp->inp_route.ro_rt); 1094 ip_freemoptions(inp->inp_moptions); 1095 kfree(inp, M_PCB); 1096 } 1097 1098 /* 1099 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP 1100 * socket received RST. 1101 */ 1102 static int 1103 in_setsockaddr(struct socket *so, struct sockaddr **nam) 1104 { 1105 struct inpcb *inp; 1106 struct sockaddr_in *sin; 1107 1108 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 1109 inp = so->so_pcb; 1110 if (!inp) 1111 return (ECONNRESET); 1112 1113 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO); 1114 sin->sin_family = AF_INET; 1115 sin->sin_len = sizeof *sin; 1116 sin->sin_port = inp->inp_lport; 1117 sin->sin_addr = inp->inp_laddr; 1118 1119 *nam = (struct sockaddr *)sin; 1120 return (0); 1121 } 1122 1123 void 1124 in_setsockaddr_dispatch(netmsg_t msg) 1125 { 1126 int error; 1127 1128 error = in_setsockaddr(msg->base.nm_so, msg->peeraddr.nm_nam); 1129 lwkt_replymsg(&msg->lmsg, error); 1130 } 1131 1132 /* 1133 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP 1134 * socket received RST. 1135 */ 1136 int 1137 in_setpeeraddr(struct socket *so, struct sockaddr **nam) 1138 { 1139 struct inpcb *inp; 1140 struct sockaddr_in *sin; 1141 1142 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 1143 inp = so->so_pcb; 1144 if (!inp) 1145 return (ECONNRESET); 1146 1147 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO); 1148 sin->sin_family = AF_INET; 1149 sin->sin_len = sizeof *sin; 1150 sin->sin_port = inp->inp_fport; 1151 sin->sin_addr = inp->inp_faddr; 1152 1153 *nam = (struct sockaddr *)sin; 1154 return (0); 1155 } 1156 1157 void 1158 in_setpeeraddr_dispatch(netmsg_t msg) 1159 { 1160 int error; 1161 1162 error = in_setpeeraddr(msg->base.nm_so, msg->peeraddr.nm_nam); 1163 lwkt_replymsg(&msg->lmsg, error); 1164 } 1165 1166 void 1167 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int err, 1168 inp_notify_t notify) 1169 { 1170 struct inpcb *inp, *marker; 1171 1172 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1173 ("not in the correct netisr")); 1174 marker = &in_pcbmarkers[mycpuid]; 1175 1176 /* 1177 * NOTE: 1178 * - If INP_PLACEMARKER is set we must ignore the rest of the 1179 * structure and skip it. 1180 * - It is safe to nuke inpcbs here, since we are in their own 1181 * netisr. 1182 */ 1183 GET_PCBINFO_TOKEN(pcbinfo); 1184 1185 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 1186 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) { 1187 LIST_REMOVE(marker, inp_list); 1188 LIST_INSERT_AFTER(inp, marker, inp_list); 1189 1190 if (inp->inp_flags & INP_PLACEMARKER) 1191 continue; 1192 #ifdef INET6 1193 if (!INP_ISIPV4(inp)) 1194 continue; 1195 #endif 1196 if (inp->inp_faddr.s_addr != faddr.s_addr || 1197 inp->inp_socket == NULL) 1198 continue; 1199 (*notify)(inp, err); /* can remove inp from list! */ 1200 } 1201 LIST_REMOVE(marker, inp_list); 1202 1203 REL_PCBINFO_TOKEN(pcbinfo); 1204 } 1205 1206 void 1207 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp) 1208 { 1209 struct inpcb *inp, *marker; 1210 1211 /* 1212 * We only need to make sure that we are in netisr0, where all 1213 * multicast operation happen. We could check inpcbinfo which 1214 * does not belong to netisr0 by holding the inpcbinfo's token. 1215 * In this case, the pcbinfo must be able to be shared, i.e. 1216 * pcbinfo->infotoken is not NULL. 1217 */ 1218 ASSERT_IN_NETISR(0); 1219 KASSERT(pcbinfo->cpu == 0 || pcbinfo->infotoken != NULL, 1220 ("pcbinfo could not be shared")); 1221 1222 /* 1223 * Get a marker for the current netisr (netisr0). 1224 * 1225 * It is possible that the multicast address deletion blocks, 1226 * which could cause temporary token releasing. So we use 1227 * inpcb marker here to get a coherent view of the inpcb list. 1228 * 1229 * While, on the other hand, moptions are only added and deleted 1230 * in netisr0, so we would not see staled moption or miss moption 1231 * even if the token was released due to the blocking multicast 1232 * address deletion. 1233 */ 1234 marker = &in_pcbmarkers[mycpuid]; 1235 1236 GET_PCBINFO_TOKEN(pcbinfo); 1237 1238 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 1239 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) { 1240 struct ip_moptions *imo; 1241 1242 LIST_REMOVE(marker, inp_list); 1243 LIST_INSERT_AFTER(inp, marker, inp_list); 1244 1245 if (inp->inp_flags & INP_PLACEMARKER) 1246 continue; 1247 imo = inp->inp_moptions; 1248 if (INP_ISIPV4(inp) && imo != NULL) { 1249 int i, gap; 1250 1251 /* 1252 * Unselect the outgoing interface if it is being 1253 * detached. 1254 */ 1255 if (imo->imo_multicast_ifp == ifp) 1256 imo->imo_multicast_ifp = NULL; 1257 1258 /* 1259 * Drop multicast group membership if we joined 1260 * through the interface being detached. 1261 */ 1262 for (i = 0, gap = 0; i < imo->imo_num_memberships; 1263 i++) { 1264 if (imo->imo_membership[i]->inm_ifp == ifp) { 1265 /* 1266 * NOTE: 1267 * This could block and the pcbinfo 1268 * token could be passively released. 1269 */ 1270 in_delmulti(imo->imo_membership[i]); 1271 gap++; 1272 } else if (gap != 0) 1273 imo->imo_membership[i - gap] = 1274 imo->imo_membership[i]; 1275 } 1276 imo->imo_num_memberships -= gap; 1277 } 1278 } 1279 LIST_REMOVE(marker, inp_list); 1280 1281 REL_PCBINFO_TOKEN(pcbinfo); 1282 } 1283 1284 /* 1285 * Check for alternatives when higher level complains 1286 * about service problems. For now, invalidate cached 1287 * routing information. If the route was created dynamically 1288 * (by a redirect), time to try a default gateway again. 1289 */ 1290 void 1291 in_losing(struct inpcb *inp) 1292 { 1293 struct rtentry *rt; 1294 struct rt_addrinfo rtinfo; 1295 1296 if ((rt = inp->inp_route.ro_rt)) { 1297 bzero(&rtinfo, sizeof(struct rt_addrinfo)); 1298 rtinfo.rti_info[RTAX_DST] = rt_key(rt); 1299 rtinfo.rti_info[RTAX_GATEWAY] = rt->rt_gateway; 1300 rtinfo.rti_info[RTAX_NETMASK] = rt_mask(rt); 1301 rtinfo.rti_flags = rt->rt_flags; 1302 rt_missmsg(RTM_LOSING, &rtinfo, rt->rt_flags, 0); 1303 if (rt->rt_flags & RTF_DYNAMIC) { 1304 rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway, 1305 rt_mask(rt), rt->rt_flags, NULL); 1306 } 1307 inp->inp_route.ro_rt = NULL; 1308 rtfree(rt); 1309 /* 1310 * A new route can be allocated 1311 * the next time output is attempted. 1312 */ 1313 } 1314 } 1315 1316 /* 1317 * After a routing change, flush old routing 1318 * and allocate a (hopefully) better one. 1319 */ 1320 void 1321 in_rtchange(struct inpcb *inp, int err) 1322 { 1323 if (inp->inp_route.ro_rt) { 1324 rtfree(inp->inp_route.ro_rt); 1325 inp->inp_route.ro_rt = NULL; 1326 /* 1327 * A new route can be allocated the next time 1328 * output is attempted. 1329 */ 1330 } 1331 } 1332 1333 /* 1334 * Lookup a PCB based on the local address and port. 1335 */ 1336 struct inpcb * 1337 in_pcblookup_local(struct inpcbportinfo *portinfo, struct in_addr laddr, 1338 u_int lport_arg, int wild_okay, struct ucred *cred) 1339 { 1340 struct inpcb *inp; 1341 int matchwild = 3, wildcard; 1342 u_short lport = lport_arg; 1343 struct inpcbporthead *porthash; 1344 struct inpcbport *phd; 1345 struct inpcb *match = NULL; 1346 1347 /* 1348 * If the porthashbase is shared across several cpus, it must 1349 * have been locked. 1350 */ 1351 ASSERT_PORT_TOKEN_HELD(portinfo); 1352 1353 /* 1354 * Best fit PCB lookup. 1355 * 1356 * First see if this local port is in use by looking on the 1357 * port hash list. 1358 */ 1359 porthash = &portinfo->porthashbase[ 1360 INP_PCBPORTHASH(lport, portinfo->porthashmask)]; 1361 LIST_FOREACH(phd, porthash, phd_hash) { 1362 if (phd->phd_port == lport) 1363 break; 1364 } 1365 if (phd != NULL) { 1366 /* 1367 * Port is in use by one or more PCBs. Look for best 1368 * fit. 1369 */ 1370 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) { 1371 wildcard = 0; 1372 #ifdef INET6 1373 if (!INP_ISIPV4(inp)) 1374 continue; 1375 #endif 1376 if (inp->inp_faddr.s_addr != INADDR_ANY) 1377 wildcard++; 1378 if (inp->inp_laddr.s_addr != INADDR_ANY) { 1379 if (laddr.s_addr == INADDR_ANY) 1380 wildcard++; 1381 else if (inp->inp_laddr.s_addr != laddr.s_addr) 1382 continue; 1383 } else { 1384 if (laddr.s_addr != INADDR_ANY) 1385 wildcard++; 1386 } 1387 if (wildcard && !wild_okay) 1388 continue; 1389 if (wildcard < matchwild && 1390 (cred == NULL || 1391 cred->cr_prison == 1392 inp->inp_socket->so_cred->cr_prison)) { 1393 match = inp; 1394 matchwild = wildcard; 1395 if (matchwild == 0) { 1396 break; 1397 } 1398 } 1399 } 1400 } 1401 return (match); 1402 } 1403 1404 struct inpcb * 1405 in_pcblocalgroup_last(const struct inpcbinfo *pcbinfo, 1406 const struct inpcb *inp) 1407 { 1408 const struct inp_localgrphead *hdr; 1409 const struct inp_localgroup *grp; 1410 int i; 1411 1412 if (pcbinfo->localgrphashbase == NULL) 1413 return NULL; 1414 1415 GET_PCBINFO_TOKEN(pcbinfo); 1416 1417 hdr = &pcbinfo->localgrphashbase[ 1418 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 1419 1420 LIST_FOREACH(grp, hdr, il_list) { 1421 if (grp->il_af == inp->inp_af && 1422 grp->il_lport == inp->inp_lport && 1423 memcmp(&grp->il_dependladdr, 1424 &inp->inp_inc.inc_ie.ie_dependladdr, 1425 sizeof(grp->il_dependladdr)) == 0) { 1426 break; 1427 } 1428 } 1429 if (grp == NULL || grp->il_inpcnt == 1) { 1430 REL_PCBINFO_TOKEN(pcbinfo); 1431 return NULL; 1432 } 1433 1434 KASSERT(grp->il_inpcnt >= 2, 1435 ("invalid localgroup inp count %d", grp->il_inpcnt)); 1436 for (i = 0; i < grp->il_inpcnt; ++i) { 1437 if (grp->il_inp[i] == inp) { 1438 int last = grp->il_inpcnt - 1; 1439 1440 if (i == last) 1441 last = grp->il_inpcnt - 2; 1442 REL_PCBINFO_TOKEN(pcbinfo); 1443 return grp->il_inp[last]; 1444 } 1445 } 1446 REL_PCBINFO_TOKEN(pcbinfo); 1447 return NULL; 1448 } 1449 1450 static struct inpcb * 1451 inp_localgroup_lookup(const struct inpcbinfo *pcbinfo, 1452 struct in_addr laddr, uint16_t lport, uint32_t pkt_hash) 1453 { 1454 struct inpcb *local_wild = NULL; 1455 const struct inp_localgrphead *hdr; 1456 const struct inp_localgroup *grp; 1457 1458 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 1459 1460 hdr = &pcbinfo->localgrphashbase[ 1461 INP_PCBLOCALGRPHASH(lport, pcbinfo->localgrphashmask)]; 1462 1463 /* 1464 * Order of socket selection: 1465 * 1. non-wild. 1466 * 2. wild. 1467 * 1468 * NOTE: Local group does not contain jailed sockets 1469 */ 1470 LIST_FOREACH(grp, hdr, il_list) { 1471 #ifdef INET6 1472 if (grp->il_af != AF_INET) 1473 continue; 1474 #endif 1475 if (grp->il_lport == lport) { 1476 int idx; 1477 1478 /* 1479 * Modulo-N is used here, which greatly reduces 1480 * completion queue token contention, thus more 1481 * cpu time is saved. 1482 */ 1483 idx = pkt_hash % grp->il_inpcnt; 1484 if (grp->il_laddr.s_addr == laddr.s_addr) 1485 return grp->il_inp[idx]; 1486 else if (grp->il_laddr.s_addr == INADDR_ANY) 1487 local_wild = grp->il_inp[idx]; 1488 } 1489 } 1490 if (local_wild != NULL) 1491 return local_wild; 1492 return NULL; 1493 } 1494 1495 /* 1496 * Lookup PCB in hash list. 1497 */ 1498 struct inpcb * 1499 in_pcblookup_pkthash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 1500 u_int fport_arg, struct in_addr laddr, u_int lport_arg, 1501 boolean_t wildcard, struct ifnet *ifp, const struct mbuf *m) 1502 { 1503 struct inpcbhead *head; 1504 struct inpcb *inp, *jinp=NULL; 1505 u_short fport = fport_arg, lport = lport_arg; 1506 1507 /* 1508 * First look for an exact match. 1509 */ 1510 head = &pcbinfo->hashbase[INP_PCBCONNHASH(faddr.s_addr, fport, 1511 laddr.s_addr, lport, pcbinfo->hashmask)]; 1512 LIST_FOREACH(inp, head, inp_hash) { 1513 #ifdef INET6 1514 if (!INP_ISIPV4(inp)) 1515 continue; 1516 #endif 1517 if (in_hosteq(inp->inp_faddr, faddr) && 1518 in_hosteq(inp->inp_laddr, laddr) && 1519 inp->inp_fport == fport && inp->inp_lport == lport) { 1520 /* found */ 1521 if (inp->inp_socket == NULL || 1522 inp->inp_socket->so_cred->cr_prison == NULL) { 1523 return (inp); 1524 } else { 1525 if (jinp == NULL) 1526 jinp = inp; 1527 } 1528 } 1529 } 1530 if (jinp != NULL) 1531 return (jinp); 1532 1533 if (wildcard) { 1534 struct inpcb *local_wild = NULL; 1535 struct inpcb *jinp_wild = NULL; 1536 struct inpcontainer *ic; 1537 struct inpcontainerhead *chead; 1538 struct sockaddr_in jsin; 1539 struct ucred *cred; 1540 1541 GET_PCBINFO_TOKEN(pcbinfo); 1542 1543 /* 1544 * Check local group first 1545 */ 1546 if (pcbinfo->localgrphashbase != NULL && 1547 m != NULL && (m->m_flags & M_HASH) && 1548 !(ifp && ifp->if_type == IFT_FAITH)) { 1549 inp = inp_localgroup_lookup(pcbinfo, 1550 laddr, lport, m->m_pkthdr.hash); 1551 if (inp != NULL) { 1552 REL_PCBINFO_TOKEN(pcbinfo); 1553 return inp; 1554 } 1555 } 1556 1557 /* 1558 * Order of socket selection: 1559 * 1. non-jailed, non-wild. 1560 * 2. non-jailed, wild. 1561 * 3. jailed, non-wild. 1562 * 4. jailed, wild. 1563 */ 1564 jsin.sin_family = AF_INET; 1565 chead = &pcbinfo->wildcardhashbase[ 1566 INP_PCBWILDCARDHASH(lport, pcbinfo->wildcardhashmask)]; 1567 LIST_FOREACH(ic, chead, ic_list) { 1568 inp = ic->ic_inp; 1569 if (inp->inp_flags & INP_PLACEMARKER) 1570 continue; 1571 1572 jsin.sin_addr.s_addr = laddr.s_addr; 1573 #ifdef INET6 1574 if (!INP_ISIPV4(inp)) 1575 continue; 1576 #endif 1577 if (inp->inp_socket != NULL) 1578 cred = inp->inp_socket->so_cred; 1579 else 1580 cred = NULL; 1581 if (cred != NULL && jailed(cred)) { 1582 if (jinp != NULL) 1583 continue; 1584 else 1585 if (!jailed_ip(cred->cr_prison, 1586 (struct sockaddr *)&jsin)) 1587 continue; 1588 } 1589 if (inp->inp_lport == lport) { 1590 if (ifp && ifp->if_type == IFT_FAITH && 1591 !(inp->inp_flags & INP_FAITH)) 1592 continue; 1593 if (inp->inp_laddr.s_addr == laddr.s_addr) { 1594 if (cred != NULL && jailed(cred)) { 1595 jinp = inp; 1596 } else { 1597 REL_PCBINFO_TOKEN(pcbinfo); 1598 return (inp); 1599 } 1600 } 1601 if (inp->inp_laddr.s_addr == INADDR_ANY) { 1602 if (cred != NULL && jailed(cred)) 1603 jinp_wild = inp; 1604 else 1605 local_wild = inp; 1606 } 1607 } 1608 } 1609 1610 REL_PCBINFO_TOKEN(pcbinfo); 1611 1612 if (local_wild != NULL) 1613 return (local_wild); 1614 if (jinp != NULL) 1615 return (jinp); 1616 return (jinp_wild); 1617 } 1618 1619 /* 1620 * Not found. 1621 */ 1622 return (NULL); 1623 } 1624 1625 struct inpcb * 1626 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr, 1627 u_int fport_arg, struct in_addr laddr, u_int lport_arg, 1628 boolean_t wildcard, struct ifnet *ifp) 1629 { 1630 return in_pcblookup_pkthash(pcbinfo, faddr, fport_arg, 1631 laddr, lport_arg, wildcard, ifp, NULL); 1632 } 1633 1634 /* 1635 * Insert PCB into connection hash table. 1636 */ 1637 void 1638 in_pcbinsconnhash(struct inpcb *inp) 1639 { 1640 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1641 struct inpcbhead *bucket; 1642 u_int32_t hashkey_faddr, hashkey_laddr; 1643 1644 #ifdef INET6 1645 if (INP_ISIPV6(inp)) { 1646 hashkey_faddr = inp->in6p_faddr.s6_addr32[3] /* XXX JH */; 1647 hashkey_laddr = inp->in6p_laddr.s6_addr32[3] /* XXX JH */; 1648 } else { 1649 #endif 1650 hashkey_faddr = inp->inp_faddr.s_addr; 1651 hashkey_laddr = inp->inp_laddr.s_addr; 1652 #ifdef INET6 1653 } 1654 #endif 1655 1656 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1657 ("not in the correct netisr")); 1658 ASSERT_INP_NOTINHASH(inp); 1659 inp->inp_flags |= INP_CONNECTED; 1660 1661 /* 1662 * Insert into the connection hash table. 1663 */ 1664 bucket = &pcbinfo->hashbase[INP_PCBCONNHASH(hashkey_faddr, 1665 inp->inp_fport, hashkey_laddr, inp->inp_lport, pcbinfo->hashmask)]; 1666 LIST_INSERT_HEAD(bucket, inp, inp_hash); 1667 } 1668 1669 /* 1670 * Remove PCB from connection hash table. 1671 */ 1672 void 1673 in_pcbremconnhash(struct inpcb *inp) 1674 { 1675 struct inpcbinfo *pcbinfo __debugvar = inp->inp_pcbinfo; 1676 1677 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 1678 ("not in the correct netisr")); 1679 KASSERT(inp->inp_flags & INP_CONNECTED, ("inp not connected")); 1680 1681 LIST_REMOVE(inp, inp_hash); 1682 inp->inp_flags &= ~INP_CONNECTED; 1683 } 1684 1685 /* 1686 * Insert PCB into port hash table. 1687 */ 1688 void 1689 in_pcbinsporthash(struct inpcbportinfo *portinfo, struct inpcb *inp) 1690 { 1691 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1692 struct inpcbporthead *pcbporthash; 1693 struct inpcbport *phd; 1694 1695 /* 1696 * If the porthashbase is shared across several cpus, it must 1697 * have been locked. 1698 */ 1699 ASSERT_PORT_TOKEN_HELD(portinfo); 1700 1701 /* 1702 * Insert into the port hash table. 1703 */ 1704 pcbporthash = &portinfo->porthashbase[ 1705 INP_PCBPORTHASH(inp->inp_lport, portinfo->porthashmask)]; 1706 1707 /* Go through port list and look for a head for this lport. */ 1708 LIST_FOREACH(phd, pcbporthash, phd_hash) { 1709 if (phd->phd_port == inp->inp_lport) 1710 break; 1711 } 1712 1713 /* If none exists, use saved one and tack it on. */ 1714 if (phd == NULL) { 1715 KKASSERT(pcbinfo->portsave != NULL); 1716 phd = pcbinfo->portsave; 1717 pcbinfo->portsave = NULL; 1718 phd->phd_port = inp->inp_lport; 1719 LIST_INIT(&phd->phd_pcblist); 1720 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash); 1721 } 1722 1723 inp->inp_portinfo = portinfo; 1724 inp->inp_phd = phd; 1725 LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist); 1726 1727 /* 1728 * Malloc one inpcbport for later use. It is safe to use 1729 * "wait" malloc here (port token would be released, if 1730 * malloc ever blocked), since all changes to the porthash 1731 * are done. 1732 */ 1733 if (pcbinfo->portsave == NULL) { 1734 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), 1735 M_PCB, M_INTWAIT | M_ZERO); 1736 } 1737 } 1738 1739 void 1740 in_pcbinsporthash_lport(struct inpcb *inp) 1741 { 1742 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 1743 struct inpcbportinfo *portinfo; 1744 u_short lport_ho; 1745 1746 /* Locate the proper portinfo based on lport */ 1747 lport_ho = ntohs(inp->inp_lport); 1748 portinfo = &pcbinfo->portinfo[lport_ho & pcbinfo->portinfo_mask]; 1749 KKASSERT((lport_ho & pcbinfo->portinfo_mask) == portinfo->offset); 1750 1751 GET_PORT_TOKEN(portinfo); 1752 in_pcbinsporthash(portinfo, inp); 1753 REL_PORT_TOKEN(portinfo); 1754 } 1755 1756 void 1757 in_pcbremporthash(struct inpcb *inp) 1758 { 1759 struct inpcbportinfo *portinfo; 1760 struct inpcbport *phd; 1761 1762 if (inp->inp_phd == NULL) 1763 return; 1764 KASSERT(inp->inp_lport != 0, ("inpcb has no lport")); 1765 1766 /* 1767 * NOTE: 1768 * inp->inp_portinfo is _not_ necessary same as 1769 * inp->inp_pcbinfo->portinfo. 1770 */ 1771 portinfo = inp->inp_portinfo; 1772 GET_PORT_TOKEN(portinfo); 1773 1774 phd = inp->inp_phd; 1775 LIST_REMOVE(inp, inp_portlist); 1776 if (LIST_FIRST(&phd->phd_pcblist) == NULL) { 1777 LIST_REMOVE(phd, phd_hash); 1778 kfree(phd, M_PCB); 1779 } 1780 1781 REL_PORT_TOKEN(portinfo); 1782 1783 inp->inp_phd = NULL; 1784 } 1785 1786 static struct inp_localgroup * 1787 inp_localgroup_alloc(u_char af, uint16_t port, 1788 const union in_dependaddr *addr, int size) 1789 { 1790 struct inp_localgroup *grp; 1791 1792 grp = kmalloc(__offsetof(struct inp_localgroup, il_inp[size]), 1793 M_TEMP, M_INTWAIT | M_ZERO); 1794 grp->il_af = af; 1795 grp->il_lport = port; 1796 grp->il_dependladdr = *addr; 1797 grp->il_inpsiz = size; 1798 1799 return grp; 1800 } 1801 1802 static void 1803 inp_localgroup_free(struct inp_localgroup *grp) 1804 { 1805 kfree(grp, M_TEMP); 1806 } 1807 1808 static void 1809 inp_localgroup_destroy(struct inp_localgroup *grp) 1810 { 1811 LIST_REMOVE(grp, il_list); 1812 inp_localgroup_free(grp); 1813 } 1814 1815 static void 1816 inp_localgroup_copy(struct inp_localgroup *grp, 1817 const struct inp_localgroup *old_grp) 1818 { 1819 int i; 1820 1821 KASSERT(old_grp->il_inpcnt < grp->il_inpsiz, 1822 ("invalid new local group size %d and old local group count %d", 1823 grp->il_inpsiz, old_grp->il_inpcnt)); 1824 for (i = 0; i < old_grp->il_inpcnt; ++i) 1825 grp->il_inp[i] = old_grp->il_inp[i]; 1826 grp->il_inpcnt = old_grp->il_inpcnt; 1827 } 1828 1829 static void 1830 in_pcbinslocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 1831 { 1832 struct inp_localgrphead *hdr; 1833 struct inp_localgroup *grp, *grp_alloc = NULL; 1834 struct ucred *cred; 1835 int i, idx; 1836 1837 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 1838 1839 if (pcbinfo->localgrphashbase == NULL) 1840 return; 1841 1842 /* 1843 * XXX don't allow jailed socket to join local group 1844 */ 1845 if (inp->inp_socket != NULL) 1846 cred = inp->inp_socket->so_cred; 1847 else 1848 cred = NULL; 1849 if (cred != NULL && jailed(cred)) 1850 return; 1851 1852 hdr = &pcbinfo->localgrphashbase[ 1853 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 1854 1855 again: 1856 LIST_FOREACH(grp, hdr, il_list) { 1857 if (grp->il_af == inp->inp_af && 1858 grp->il_lport == inp->inp_lport && 1859 memcmp(&grp->il_dependladdr, 1860 &inp->inp_inc.inc_ie.ie_dependladdr, 1861 sizeof(grp->il_dependladdr)) == 0) { 1862 break; 1863 } 1864 } 1865 if (grp == NULL) { 1866 /* 1867 * Create a new local group 1868 */ 1869 if (grp_alloc == NULL) { 1870 grp_alloc = inp_localgroup_alloc(inp->inp_af, 1871 inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr, 1872 INP_LOCALGROUP_SIZMIN); 1873 /* 1874 * Local group allocation could block and the 1875 * local group w/ the same property might have 1876 * been added by others when we were blocked; 1877 * check again. 1878 */ 1879 goto again; 1880 } else { 1881 /* Local group has been allocated; link it */ 1882 grp = grp_alloc; 1883 grp_alloc = NULL; 1884 LIST_INSERT_HEAD(hdr, grp, il_list); 1885 } 1886 } else if (grp->il_inpcnt == grp->il_inpsiz) { 1887 if (grp->il_inpsiz >= INP_LOCALGROUP_SIZMAX) { 1888 static int limit_logged = 0; 1889 1890 if (!limit_logged) { 1891 limit_logged = 1; 1892 kprintf("local group port %d, " 1893 "limit reached\n", ntohs(grp->il_lport)); 1894 } 1895 if (grp_alloc != NULL) { 1896 /* 1897 * This would happen if the local group 1898 * w/ the same property was expanded when 1899 * our local group allocation blocked. 1900 */ 1901 inp_localgroup_free(grp_alloc); 1902 } 1903 return; 1904 } 1905 1906 /* 1907 * Expand this local group 1908 */ 1909 if (grp_alloc == NULL || 1910 grp->il_inpcnt >= grp_alloc->il_inpsiz) { 1911 if (grp_alloc != NULL) 1912 inp_localgroup_free(grp_alloc); 1913 grp_alloc = inp_localgroup_alloc(grp->il_af, 1914 grp->il_lport, &grp->il_dependladdr, 1915 grp->il_inpsiz * 2); 1916 /* 1917 * Local group allocation could block and the 1918 * local group w/ the same property might have 1919 * been expanded by others when we were blocked; 1920 * check again. 1921 */ 1922 goto again; 1923 } 1924 1925 /* 1926 * Save the old local group, link the new one, and then 1927 * destroy the old local group 1928 */ 1929 inp_localgroup_copy(grp_alloc, grp); 1930 LIST_INSERT_HEAD(hdr, grp_alloc, il_list); 1931 inp_localgroup_destroy(grp); 1932 1933 grp = grp_alloc; 1934 grp_alloc = NULL; 1935 } else { 1936 /* 1937 * Found the local group 1938 */ 1939 if (grp_alloc != NULL) { 1940 /* 1941 * This would happen if the local group w/ the 1942 * same property was added or expanded when our 1943 * local group allocation blocked. 1944 */ 1945 inp_localgroup_free(grp_alloc); 1946 grp_alloc = NULL; 1947 } 1948 } 1949 1950 KASSERT(grp->il_inpcnt < grp->il_inpsiz, 1951 ("invalid local group size %d and count %d", 1952 grp->il_inpsiz, grp->il_inpcnt)); 1953 1954 /* 1955 * Keep the local group sorted by the inpcb local group index 1956 * in ascending order. 1957 * 1958 * This eases the multi-process userland application which uses 1959 * SO_REUSEPORT sockets and binds process to the owner cpu of 1960 * the SO_REUSEPORT socket: 1961 * If we didn't sort the local group by the inpcb local group 1962 * index and one of the process owning an inpcb in this local 1963 * group restarted, e.g. crashed and restarted by watchdog, 1964 * other processes owning a inpcb in this local group would have 1965 * to detect that event, refetch its socket's owner cpu, and 1966 * re-bind. 1967 */ 1968 idx = grp->il_inpcnt; 1969 for (i = 0; i < idx; ++i) { 1970 struct inpcb *oinp = grp->il_inp[i]; 1971 1972 if (oinp->inp_lgrpindex > i) { 1973 if (inp->inp_lgrpindex < 0) { 1974 inp->inp_lgrpindex = i; 1975 } else if (inp->inp_lgrpindex != i) { 1976 if (bootverbose) { 1977 kprintf("inp %p: grpidx %d, " 1978 "assigned to %d, cpu%d\n", 1979 inp, inp->inp_lgrpindex, i, 1980 mycpuid); 1981 } 1982 } 1983 grp->il_inp[i] = inp; 1984 1985 /* Pull down inpcbs */ 1986 for (; i < grp->il_inpcnt; ++i) { 1987 struct inpcb *oinp1 = grp->il_inp[i + 1]; 1988 1989 grp->il_inp[i + 1] = oinp; 1990 oinp = oinp1; 1991 } 1992 grp->il_inpcnt++; 1993 return; 1994 } 1995 } 1996 1997 if (inp->inp_lgrpindex < 0) { 1998 inp->inp_lgrpindex = idx; 1999 } else if (inp->inp_lgrpindex != idx) { 2000 if (bootverbose) { 2001 kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n", 2002 inp, inp->inp_lgrpindex, idx, mycpuid); 2003 } 2004 } 2005 grp->il_inp[idx] = inp; 2006 grp->il_inpcnt++; 2007 } 2008 2009 void 2010 in_pcbinswildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2011 { 2012 struct inpcontainer *ic; 2013 struct inpcontainerhead *bucket; 2014 2015 GET_PCBINFO_TOKEN(pcbinfo); 2016 2017 in_pcbinslocalgrphash_oncpu(inp, pcbinfo); 2018 2019 bucket = &pcbinfo->wildcardhashbase[ 2020 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)]; 2021 2022 ic = kmalloc(sizeof(struct inpcontainer), M_TEMP, M_INTWAIT); 2023 ic->ic_inp = inp; 2024 LIST_INSERT_HEAD(bucket, ic, ic_list); 2025 2026 REL_PCBINFO_TOKEN(pcbinfo); 2027 } 2028 2029 /* 2030 * Insert PCB into wildcard hash table. 2031 */ 2032 void 2033 in_pcbinswildcardhash(struct inpcb *inp) 2034 { 2035 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2036 2037 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 2038 ("not in correct netisr")); 2039 ASSERT_INP_NOTINHASH(inp); 2040 inp->inp_flags |= INP_WILDCARD; 2041 2042 in_pcbinswildcardhash_oncpu(inp, pcbinfo); 2043 } 2044 2045 static void 2046 in_pcbremlocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2047 { 2048 struct inp_localgrphead *hdr; 2049 struct inp_localgroup *grp; 2050 2051 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo); 2052 2053 if (pcbinfo->localgrphashbase == NULL) 2054 return; 2055 2056 hdr = &pcbinfo->localgrphashbase[ 2057 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)]; 2058 2059 LIST_FOREACH(grp, hdr, il_list) { 2060 int i; 2061 2062 for (i = 0; i < grp->il_inpcnt; ++i) { 2063 if (grp->il_inp[i] != inp) 2064 continue; 2065 2066 if (grp->il_inpcnt == 1) { 2067 /* Destroy this local group */ 2068 inp_localgroup_destroy(grp); 2069 } else { 2070 /* Pull up inpcbs */ 2071 for (; i + 1 < grp->il_inpcnt; ++i) 2072 grp->il_inp[i] = grp->il_inp[i + 1]; 2073 grp->il_inpcnt--; 2074 } 2075 return; 2076 } 2077 } 2078 } 2079 2080 void 2081 in_pcbremwildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo) 2082 { 2083 struct inpcontainer *ic; 2084 struct inpcontainerhead *head; 2085 2086 GET_PCBINFO_TOKEN(pcbinfo); 2087 2088 in_pcbremlocalgrphash_oncpu(inp, pcbinfo); 2089 2090 /* find bucket */ 2091 head = &pcbinfo->wildcardhashbase[ 2092 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)]; 2093 2094 LIST_FOREACH(ic, head, ic_list) { 2095 if (ic->ic_inp == inp) 2096 goto found; 2097 } 2098 REL_PCBINFO_TOKEN(pcbinfo); 2099 return; /* not found! */ 2100 2101 found: 2102 LIST_REMOVE(ic, ic_list); /* remove container from bucket chain */ 2103 REL_PCBINFO_TOKEN(pcbinfo); 2104 kfree(ic, M_TEMP); /* deallocate container */ 2105 } 2106 2107 /* 2108 * Remove PCB from wildcard hash table. 2109 */ 2110 void 2111 in_pcbremwildcardhash(struct inpcb *inp) 2112 { 2113 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; 2114 2115 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu), 2116 ("not in correct netisr")); 2117 KASSERT(inp->inp_flags & INP_WILDCARD, ("inp not wildcard")); 2118 2119 in_pcbremwildcardhash_oncpu(inp, pcbinfo); 2120 inp->inp_lgrpindex = -1; 2121 inp->inp_flags &= ~INP_WILDCARD; 2122 } 2123 2124 /* 2125 * Remove PCB from various lists. 2126 */ 2127 void 2128 in_pcbremlists(struct inpcb *inp) 2129 { 2130 in_pcbremporthash(inp); 2131 if (inp->inp_flags & INP_WILDCARD) { 2132 in_pcbremwildcardhash(inp); 2133 } else if (inp->inp_flags & INP_CONNECTED) { 2134 in_pcbremconnhash(inp); 2135 } 2136 2137 if (inp->inp_flags & INP_ONLIST) 2138 in_pcbofflist(inp); 2139 } 2140 2141 int 2142 prison_xinpcb(struct thread *td, struct inpcb *inp) 2143 { 2144 struct ucred *cr; 2145 2146 if (td->td_proc == NULL) 2147 return (0); 2148 cr = td->td_proc->p_ucred; 2149 if (cr->cr_prison == NULL) 2150 return (0); 2151 if (inp->inp_socket && inp->inp_socket->so_cred && 2152 inp->inp_socket->so_cred->cr_prison && 2153 cr->cr_prison == inp->inp_socket->so_cred->cr_prison) 2154 return (0); 2155 return (1); 2156 } 2157 2158 int 2159 in_pcblist_global(SYSCTL_HANDLER_ARGS) 2160 { 2161 struct inpcbinfo *pcbinfo_arr = arg1; 2162 int pcbinfo_arrlen = arg2; 2163 struct inpcb *marker; 2164 int cpu, origcpu; 2165 int error, n; 2166 2167 KASSERT(pcbinfo_arrlen <= ncpus && pcbinfo_arrlen >= 1, 2168 ("invalid pcbinfo count %d", pcbinfo_arrlen)); 2169 2170 /* 2171 * The process of preparing the TCB list is too time-consuming and 2172 * resource-intensive to repeat twice on every request. 2173 */ 2174 n = 0; 2175 if (req->oldptr == NULL) { 2176 for (cpu = 0; cpu < pcbinfo_arrlen; ++cpu) 2177 n += pcbinfo_arr[cpu].ipi_count; 2178 req->oldidx = (n + n/8 + 10) * sizeof(struct xinpcb); 2179 return 0; 2180 } 2181 2182 if (req->newptr != NULL) 2183 return EPERM; 2184 2185 marker = kmalloc(sizeof(struct inpcb), M_TEMP, M_WAITOK|M_ZERO); 2186 marker->inp_flags |= INP_PLACEMARKER; 2187 2188 /* 2189 * OK, now we're committed to doing something. Re-fetch ipi_count 2190 * after obtaining the generation count. 2191 */ 2192 error = 0; 2193 origcpu = mycpuid; 2194 for (cpu = 0; cpu < pcbinfo_arrlen && error == 0; ++cpu) { 2195 struct inpcbinfo *pcbinfo = &pcbinfo_arr[cpu]; 2196 struct inpcb *inp; 2197 struct xinpcb xi; 2198 int i; 2199 2200 lwkt_migratecpu(cpu); 2201 2202 GET_PCBINFO_TOKEN(pcbinfo); 2203 2204 n = pcbinfo->ipi_count; 2205 2206 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list); 2207 i = 0; 2208 while ((inp = LIST_NEXT(marker, inp_list)) != NULL && i < n) { 2209 LIST_REMOVE(marker, inp_list); 2210 LIST_INSERT_AFTER(inp, marker, inp_list); 2211 2212 if (inp->inp_flags & INP_PLACEMARKER) 2213 continue; 2214 if (prison_xinpcb(req->td, inp)) 2215 continue; 2216 2217 bzero(&xi, sizeof xi); 2218 xi.xi_len = sizeof xi; 2219 bcopy(inp, &xi.xi_inp, sizeof *inp); 2220 if (inp->inp_socket) 2221 sotoxsocket(inp->inp_socket, &xi.xi_socket); 2222 if ((error = SYSCTL_OUT(req, &xi, sizeof xi)) != 0) 2223 break; 2224 ++i; 2225 } 2226 LIST_REMOVE(marker, inp_list); 2227 2228 REL_PCBINFO_TOKEN(pcbinfo); 2229 2230 if (error == 0 && i < n) { 2231 bzero(&xi, sizeof xi); 2232 xi.xi_len = sizeof xi; 2233 while (i < n) { 2234 error = SYSCTL_OUT(req, &xi, sizeof xi); 2235 if (error) 2236 break; 2237 ++i; 2238 } 2239 } 2240 } 2241 2242 lwkt_migratecpu(origcpu); 2243 kfree(marker, M_TEMP); 2244 return error; 2245 } 2246 2247 int 2248 in_pcblist_global_ncpus2(SYSCTL_HANDLER_ARGS) 2249 { 2250 return in_pcblist_global(oidp, arg1, ncpus2, req); 2251 } 2252 2253 void 2254 in_savefaddr(struct socket *so, const struct sockaddr *faddr) 2255 { 2256 struct sockaddr_in *sin; 2257 2258 KASSERT(faddr->sa_family == AF_INET, 2259 ("not AF_INET faddr %d", faddr->sa_family)); 2260 2261 sin = kmalloc(sizeof(*sin), M_SONAME, M_WAITOK | M_ZERO); 2262 sin->sin_family = AF_INET; 2263 sin->sin_len = sizeof(*sin); 2264 sin->sin_port = ((const struct sockaddr_in *)faddr)->sin_port; 2265 sin->sin_addr = ((const struct sockaddr_in *)faddr)->sin_addr; 2266 2267 so->so_faddr = (struct sockaddr *)sin; 2268 } 2269 2270 void 2271 in_pcbportinfo_init(struct inpcbportinfo *portinfo, int hashsize, 2272 boolean_t shared, u_short offset) 2273 { 2274 memset(portinfo, 0, sizeof(*portinfo)); 2275 2276 portinfo->offset = offset; 2277 portinfo->lastport = offset; 2278 portinfo->lastlow = offset; 2279 portinfo->lasthi = offset; 2280 2281 portinfo->porthashbase = hashinit(hashsize, M_PCB, 2282 &portinfo->porthashmask); 2283 2284 if (shared) { 2285 portinfo->porttoken = kmalloc(sizeof(struct lwkt_token), 2286 M_PCB, M_WAITOK); 2287 lwkt_token_init(portinfo->porttoken, "porttoken"); 2288 } 2289 } 2290 2291 void 2292 in_pcbportrange(u_short *hi0, u_short *lo0, u_short ofs, u_short step) 2293 { 2294 int hi, lo; 2295 2296 if (step == 1) 2297 return; 2298 2299 hi = *hi0; 2300 lo = *lo0; 2301 2302 hi = rounddown2(hi, step); 2303 hi += ofs; 2304 if (hi > (int)*hi0) 2305 hi -= step; 2306 2307 lo = roundup2(lo, step); 2308 lo -= (step - ofs); 2309 if (lo < (int)*lo0) 2310 lo += step; 2311 2312 *hi0 = hi; 2313 *lo0 = lo; 2314 } 2315 2316 void 2317 in_pcbglobalinit(void) 2318 { 2319 int cpu; 2320 2321 in_pcbmarkers = kmalloc(ncpus * sizeof(struct inpcb), M_PCB, 2322 M_WAITOK | M_ZERO); 2323 in_pcbcontainer_markers = kmalloc(ncpus * sizeof(struct inpcontainer), 2324 M_PCB, M_WAITOK | M_ZERO); 2325 2326 for (cpu = 0; cpu < ncpus; ++cpu) { 2327 struct inpcontainer *ic = &in_pcbcontainer_markers[cpu]; 2328 struct inpcb *marker = &in_pcbmarkers[cpu]; 2329 2330 marker->inp_flags |= INP_PLACEMARKER; 2331 ic->ic_inp = marker; 2332 } 2333 } 2334 2335 struct inpcb * 2336 in_pcbmarker(int cpuid) 2337 { 2338 KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid)); 2339 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 2340 2341 return &in_pcbmarkers[cpuid]; 2342 } 2343 2344 struct inpcontainer * 2345 in_pcbcontainer_marker(int cpuid) 2346 { 2347 KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid %d", cpuid)); 2348 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr")); 2349 2350 return &in_pcbcontainer_markers[cpuid]; 2351 } 2352 2353 void 2354 in_pcbresetroute(struct inpcb *inp) 2355 { 2356 struct route *ro = &inp->inp_route; 2357 2358 if (ro->ro_rt != NULL) 2359 RTFREE(ro->ro_rt); 2360 bzero(ro, sizeof(*ro)); 2361 } 2362