Lines Matching defs:inp
297 in_pcblbgroup_find(struct inpcb *inp)
303 INP_LOCK_ASSERT(inp);
305 pcbinfo = inp->inp_pcbinfo;
309 INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
314 if (inp == grp->il_inp[i])
318 if (inp == inp1)
327 in_pcblbgroup_insert(struct inpcblbgroup *grp, struct inpcb *inp)
332 INP_WLOCK_ASSERT(inp);
334 if (inp->inp_socket->so_proto->pr_listen != pr_listen_notsupp &&
335 !SOLISTENING(inp->inp_socket)) {
340 LIST_INSERT_HEAD(&grp->il_pending, inp, inp_lbgroup_list);
342 grp->il_inp[grp->il_inpcnt] = inp;
351 inp->inp_flags |= INP_INLBGROUP;
385 in_pcbinslbgrouphash(struct inpcb *inp, uint8_t numa_domain)
394 pcbinfo = inp->inp_pcbinfo;
396 INP_WLOCK_ASSERT(inp);
403 if ((inp->inp_vflag & INP_IPV4) &&
404 inp->inp_laddr.s_addr == INADDR_ANY &&
405 INP_CHECK_SOCKAF(inp->inp_socket, AF_INET6)) {
410 idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask);
413 if (grp->il_cred->cr_prison == inp->inp_cred->cr_prison &&
414 grp->il_vflag == inp->inp_vflag &&
415 grp->il_lport == inp->inp_lport &&
418 &inp->inp_inc.inc_ie.ie_dependladdr,
425 grp = in_pcblbgroup_alloc(inp->inp_cred, inp->inp_vflag,
426 inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
430 in_pcblbgroup_insert(grp, inp);
444 in_pcblbgroup_insert(grp, inp);
446 in_pcblbgroup_insert(grp, inp);
455 in_pcbremlbgrouphash(struct inpcb *inp)
463 pcbinfo = inp->inp_pcbinfo;
465 INP_WLOCK_ASSERT(inp);
466 MPASS(inp->inp_flags & INP_INLBGROUP);
470 INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
473 if (grp->il_inp[i] != inp)
490 inp->inp_flags &= ~INP_INLBGROUP;
494 if (inp == inp1) {
495 LIST_REMOVE(inp, inp_lbgroup_list);
496 inp->inp_flags &= ~INP_INLBGROUP;
505 in_pcblbgroup_numa(struct inpcb *inp, int arg)
524 pcbinfo = inp->inp_pcbinfo;
525 INP_WLOCK_ASSERT(inp);
527 if (in_pcblbgroup_find(inp) != NULL) {
529 in_pcbremlbgrouphash(inp);
531 in_pcbinslbgrouphash(inp, numa_domain);
630 struct inpcb *inp;
635 inp = uma_zalloc_smr(pcbinfo->ipi_zone, M_NOWAIT);
636 if (inp == NULL)
638 bzero(&inp->inp_start_zero, inp_zero_size);
640 inp->inp_numa_domain = M_NODOM;
642 inp->inp_pcbinfo = pcbinfo;
643 inp->inp_socket = so;
644 inp->inp_cred = crhold(so->so_cred);
645 inp->inp_inc.inc_fibnum = so->so_fibnum;
647 error = mac_inpcb_init(inp, M_NOWAIT);
650 mac_inpcb_create(so, inp);
653 error = ipsec_init_pcbpolicy(inp);
656 mac_inpcb_destroy(inp);
663 inp->inp_vflag |= INP_IPV6PROTO | INP_IPV6;
665 inp->inp_flags |= IN6P_IPV6_V6ONLY;
668 inp->inp_vflag |= INP_IPV4;
671 inp->inp_flags |= IN6P_AUTOFLOWLABEL;
672 inp->in6p_hops = -1; /* use kernel default */
679 inp->inp_vflag |= INP_IPV4;
681 inp->inp_smr = SMR_SEQ_INVALID;
687 inp->inp_route.ro_flags = RT_LLE_CACHE;
688 refcount_init(&inp->inp_refcount, 1); /* Reference from socket. */
689 INP_WLOCK(inp);
692 inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
693 CK_LIST_INSERT_HEAD(&pcbinfo->ipi_listhead, inp, inp_list);
695 so->so_pcb = inp;
701 crfree(inp->inp_cred);
703 inp->inp_cred = NULL;
705 uma_zfree_smr(pcbinfo->ipi_zone, inp);
712 in_pcbbind(struct inpcb *inp, struct sockaddr_in *sin, struct ucred *cred)
720 INP_WLOCK_ASSERT(inp);
721 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
723 if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
726 error = in_pcbbind_setup(inp, sin, &inp->inp_laddr.s_addr,
727 &inp->inp_lport, cred);
730 if (in_pcbinshash(inp) != 0) {
731 inp->inp_laddr.s_addr = INADDR_ANY;
732 inp->inp_lport = 0;
736 inp->inp_flags |= INP_ANONPORT;
749 in_pcb_lport_dest(struct inpcb *inp, struct sockaddr *lsa, u_short *lportp,
764 pcbinfo = inp->inp_pcbinfo;
770 INP_LOCK_ASSERT(inp);
773 if (inp->inp_flags & INP_HIGHPORT) {
777 } else if (inp->inp_flags & INP_LOWPORT) {
803 if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) {
812 if ((inp->inp_vflag & INP_IPV6) != 0) {
853 if ((inp->inp_vflag & INP_IPV6) != 0) {
855 &inp->in6p_laddr, lport, lookupflags, cred);
858 (inp->inp_vflag & INP_IPV4))
883 in_pcb_lport(struct inpcb *inp, struct in_addr *laddrp, u_short *lportp,
893 return (in_pcb_lport_dest(inp, laddrp ? (struct sockaddr *) &laddr :
903 in_pcbbind_avail(struct inpcb *inp, const struct in_addr laddr,
908 INP_LOCK_ASSERT(inp);
909 INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
943 if ((inp->inp_flags & INP_BINDANY) == 0 &&
957 priv_check_cred(inp->inp_cred, PRIV_NETINET_REUSEPORT) != 0) {
967 t = in_pcblookup_local(inp->inp_pcbinfo, laddr, lport,
970 (inp->inp_socket->so_type != SOCK_STREAM ||
972 (inp->inp_cred->cr_uid != t->inp_cred->cr_uid))
975 t = in_pcblookup_local(inp->inp_pcbinfo, laddr, lport,
982 (inp->inp_vflag & INP_IPV6PROTO) == 0 ||
1001 in_pcbbind_setup(struct inpcb *inp, struct sockaddr_in *sin, in_addr_t *laddrp,
1004 struct socket *so = inp->inp_socket;
1013 INP_LOCK_ASSERT(inp);
1014 INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
1045 error = in_pcbbind_avail(inp, laddr, lport, sooptions,
1053 error = in_pcb_lport(inp, &laddr, &lport, cred, lookupflags);
1069 in_pcbconnect(struct inpcb *inp, struct sockaddr_in *sin, struct ucred *cred)
1075 INP_WLOCK_ASSERT(inp);
1076 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
1077 KASSERT(in_nullhost(inp->inp_faddr),
1078 ("%s: inp is already connected", __func__));
1080 lport = inp->inp_lport;
1081 laddr = inp->inp_laddr.s_addr;
1083 error = in_pcbconnect_setup(inp, sin, &laddr, &lport, &faddr, &fport,
1088 inp->inp_faddr.s_addr = faddr;
1089 inp->inp_fport = fport;
1092 if (inp->inp_laddr.s_addr == INADDR_ANY && inp->inp_lport == 0) {
1093 inp->inp_lport = lport;
1094 inp->inp_laddr.s_addr = laddr;
1095 if (in_pcbinshash(inp) != 0) {
1096 inp->inp_laddr.s_addr = inp->inp_faddr.s_addr =
1098 inp->inp_lport = inp->inp_fport = 0;
1102 inp->inp_lport = lport;
1103 inp->inp_laddr.s_addr = laddr;
1104 if ((inp->inp_flags & INP_INHASHLIST) != 0)
1105 in_pcbrehash(inp);
1107 in_pcbinshash(inp);
1111 inp->inp_flags |= INP_ANONPORT;
1120 in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr,
1154 if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0)
1155 nh = fib4_lookup(inp->inp_inc.inc_fibnum, *faddr,
1171 inp->inp_socket->so_fibnum));
1174 inp->inp_socket->so_fibnum));
1276 inp->inp_socket->so_fibnum));
1279 inp->inp_socket->so_fibnum));
1337 in_pcbconnect_setup(struct inpcb *inp, struct sockaddr_in *sin,
1356 INP_LOCK_ASSERT(inp);
1357 INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
1370 inp->inp_socket->so_proto->pr_protocol, &hash_type);
1372 inp->inp_flowid = hash_val;
1373 inp->inp_flowtype = hash_type;
1399 error = in_pcbladdr(inp, &faddr, &laddr, cred);
1406 inp->inp_moptions != NULL) {
1410 imo = inp->inp_moptions;
1432 if (in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr,
1444 error = in_pcb_lport_dest(inp, (struct sockaddr *) &lsin,
1458 in_pcbdisconnect(struct inpcb *inp)
1461 INP_WLOCK_ASSERT(inp);
1462 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
1463 KASSERT(inp->inp_smr == SMR_SEQ_INVALID,
1464 ("%s: inp %p was already disconnected", __func__, inp));
1466 in_pcbremhash_locked(inp);
1469 inp->inp_smr = smr_advance(inp->inp_pcbinfo->ipi_smr);
1470 inp->inp_laddr.s_addr = INADDR_ANY;
1471 inp->inp_faddr.s_addr = INADDR_ANY;
1472 inp->inp_fport = 0;
1477 in_pcblisten(struct inpcb *inp)
1481 INP_WLOCK_ASSERT(inp);
1483 if ((inp->inp_flags & INP_INLBGROUP) != 0) {
1486 pcbinfo = inp->inp_pcbinfo;
1488 grp = in_pcblbgroup_find(inp);
1489 LIST_REMOVE(inp, inp_lbgroup_list);
1490 in_pcblbgroup_insert(grp, inp);
1504 inp_lock(struct inpcb *inp, const inp_lookup_t lock)
1508 rw_rlock(&inp->inp_lock) : rw_wlock(&inp->inp_lock);
1512 inp_unlock(struct inpcb *inp, const inp_lookup_t lock)
1516 rw_runlock(&inp->inp_lock) : rw_wunlock(&inp->inp_lock);
1520 inp_trylock(struct inpcb *inp, const inp_lookup_t lock)
1524 rw_try_rlock(&inp->inp_lock) : rw_try_wlock(&inp->inp_lock));
1528 _inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock, const int ignflags)
1532 SMR_ASSERT_ENTERED(inp->inp_pcbinfo->ipi_smr);
1534 if (__predict_true(inp_trylock(inp, lock))) {
1535 if (__predict_false(inp->inp_flags & ignflags)) {
1536 smr_exit(inp->inp_pcbinfo->ipi_smr);
1537 inp_unlock(inp, lock);
1540 smr_exit(inp->inp_pcbinfo->ipi_smr);
1544 if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) {
1545 smr_exit(inp->inp_pcbinfo->ipi_smr);
1546 inp_lock(inp, lock);
1547 if (__predict_false(in_pcbrele(inp, lock)))
1550 * inp acquired through refcount & lock for sure didn't went
1555 if (__predict_false(inp->inp_flags & ignflags)) {
1556 inp_unlock(inp, lock);
1561 smr_exit(inp->inp_pcbinfo->ipi_smr);
1567 inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock)
1575 return (_inp_smr_lock(inp, lock, INP_FREED | INP_DROPPED));
1609 #define II_LIST_NEXT(inp, hash) \
1611 CK_LIST_NEXT((inp), inp_list) : \
1612 CK_LIST_NEXT((inp), inp_hash_exact))
1613 #define II_LOCK_ASSERT(inp, lock) \
1614 rw_assert(&(inp)->inp_lock, \
1624 struct inpcb *inp;
1626 if (ii->inp == NULL) { /* First call. */
1629 for (inp = II_LIST_FIRST(ipi, hash);
1630 inp != NULL;
1631 inp = II_LIST_NEXT(inp, hash)) {
1632 if (match != NULL && (match)(inp, ctx) == false)
1634 if (__predict_true(_inp_smr_lock(inp, lock, INP_FREED)))
1638 MPASS(inp != II_LIST_FIRST(ipi, hash));
1639 inp = II_LIST_FIRST(ipi, hash);
1640 if (inp == NULL)
1645 if (inp == NULL)
1648 ii->inp = inp;
1650 return (inp);
1656 inp = ii->inp;
1657 II_LOCK_ASSERT(inp, lock);
1659 inp = II_LIST_NEXT(inp, hash);
1660 if (inp == NULL) {
1665 if (match != NULL && (match)(inp, ctx) == false)
1668 if (__predict_true(inp_trylock(inp, lock))) {
1669 if (__predict_false(inp->inp_flags & INP_FREED)) {
1677 inp_unlock(inp, lock);
1688 * anchoring point is ii->inp, which we keep locked for this case, so
1691 if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) {
1693 inp_lock(inp, lock);
1694 if (__predict_false(in_pcbrele(inp, lock))) {
1701 if (__predict_false(inp->inp_flags & INP_FREED)) {
1702 inp_unlock(inp, lock);
1710 inp_unlock(ii->inp, lock);
1711 ii->inp = inp;
1713 return (ii->inp);
1724 in_pcbref(struct inpcb *inp)
1728 old = refcount_acquire(&inp->inp_refcount);
1737 in_pcbrele_rlocked(struct inpcb *inp)
1740 INP_RLOCK_ASSERT(inp);
1742 if (!refcount_release(&inp->inp_refcount))
1745 MPASS(inp->inp_flags & INP_FREED);
1746 MPASS(inp->inp_socket == NULL);
1747 crfree(inp->inp_cred);
1749 inp->inp_cred = NULL;
1751 INP_RUNLOCK(inp);
1752 uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp);
1757 in_pcbrele_wlocked(struct inpcb *inp)
1760 INP_WLOCK_ASSERT(inp);
1762 if (!refcount_release(&inp->inp_refcount))
1765 MPASS(inp->inp_flags & INP_FREED);
1766 MPASS(inp->inp_socket == NULL);
1767 crfree(inp->inp_cred);
1769 inp->inp_cred = NULL;
1771 INP_WUNLOCK(inp);
1772 uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp);
1777 in_pcbrele(struct inpcb *inp, const inp_lookup_t lock)
1781 in_pcbrele_rlocked(inp) : in_pcbrele_wlocked(inp));
1794 in_pcbfree(struct inpcb *inp)
1796 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1804 INP_WLOCK_ASSERT(inp);
1805 KASSERT(inp->inp_socket != NULL, ("%s: inp_socket == NULL", __func__));
1806 KASSERT((inp->inp_flags & INP_FREED) == 0,
1807 ("%s: called twice for pcb %p", __func__, inp));
1814 if (inp->inp_flags & INP_INHASHLIST)
1815 in_pcbremhash(inp);
1817 inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
1819 CK_LIST_REMOVE(inp, inp_list);
1823 if (inp->inp_snd_tag != NULL)
1824 in_pcbdetach_txrtlmt(inp);
1826 inp->inp_flags |= INP_FREED;
1827 inp->inp_socket->so_pcb = NULL;
1828 inp->inp_socket = NULL;
1830 RO_INVALIDATE_CACHE(&inp->inp_route);
1832 mac_inpcb_destroy(inp);
1835 if (inp->inp_sp != NULL)
1836 ipsec_delete_pcbpolicy(inp);
1839 if (inp->inp_options)
1840 (void)m_free(inp->inp_options);
1841 DEBUG_POISON_POINTER(inp->inp_options);
1842 imo = inp->inp_moptions;
1843 DEBUG_POISON_POINTER(inp->inp_moptions);
1846 if (inp->inp_vflag & INP_IPV6PROTO) {
1847 ip6_freepcbopts(inp->in6p_outputopts);
1848 DEBUG_POISON_POINTER(inp->in6p_outputopts);
1849 im6o = inp->in6p_moptions;
1850 DEBUG_POISON_POINTER(inp->in6p_moptions);
1855 if (__predict_false(in_pcbrele_wlocked(inp) == false)) {
1856 INP_WUNLOCK(inp);
1873 struct inpcb *inp = mem;
1875 INP_LOCK_DESTROY(inp);
1893 in_pcbdrop(struct inpcb *inp)
1896 INP_WLOCK_ASSERT(inp);
1898 inp->inp_flags |= INP_DROPPED;
1899 if (inp->inp_flags & INP_INHASHLIST)
1900 in_pcbremhash(inp);
1910 struct inpcb *inp;
1912 inp = sotoinpcb(so);
1913 KASSERT(inp != NULL, ("in_getsockaddr: inp == NULL"));
1918 .sin_port = inp->inp_lport,
1919 .sin_addr = inp->inp_laddr,
1928 struct inpcb *inp;
1930 inp = sotoinpcb(so);
1931 KASSERT(inp != NULL, ("in_getpeeraddr: inp == NULL"));
1936 .sin_port = inp->inp_fport,
1937 .sin_addr = inp->inp_faddr,
1944 inp_v4_multi_match(const struct inpcb *inp, void *v __unused)
1947 if ((inp->inp_vflag & INP_IPV4) && inp->inp_moptions != NULL)
1958 struct inpcb *inp;
1965 while ((inp = inp_next(&inpi)) != NULL) {
1966 INP_WLOCK_ASSERT(inp);
1968 imo = inp->inp_moptions;
2005 struct inpcb *inp;
2025 CK_LIST_FOREACH(inp, head, inp_hash_wild) {
2027 /* XXX inp locking */
2028 if ((inp->inp_vflag & INP_IPV4) == 0)
2031 if (inp->inp_faddr.s_addr == INADDR_ANY &&
2032 inp->inp_laddr.s_addr == laddr.s_addr &&
2033 inp->inp_lport == lport) {
2038 inp->inp_cred->cr_prison))
2039 return (inp);
2067 CK_LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
2069 if (!prison_equal_ip4(inp->inp_cred->cr_prison,
2073 /* XXX inp locking */
2074 if ((inp->inp_vflag & INP_IPV4) == 0)
2089 if ((inp->inp_vflag & INP_IPV6) != 0)
2092 if (inp->inp_faddr.s_addr != INADDR_ANY)
2094 if (inp->inp_laddr.s_addr != INADDR_ANY) {
2097 else if (inp->inp_laddr.s_addr != laddr.s_addr)
2104 match = inp;
2130 struct inpcb *inp;
2201 inp = grp->il_inp[INP_PCBLBGROUP_PKTHASH(faddr, lport, fport) % count];
2202 KASSERT(inp != NULL, ("%s: inp == NULL", __func__));
2203 return (inp);
2207 in_pcblookup_exact_match(const struct inpcb *inp, struct in_addr faddr,
2211 /* XXX inp locking */
2212 if ((inp->inp_vflag & INP_IPV4) == 0)
2215 if (inp->inp_faddr.s_addr == faddr.s_addr &&
2216 inp->inp_laddr.s_addr == laddr.s_addr &&
2217 inp->inp_fport == fport &&
2218 inp->inp_lport == lport)
2228 struct inpcb *inp;
2234 CK_LIST_FOREACH(inp, head, inp_hash_exact) {
2235 if (in_pcblookup_exact_match(inp, faddr, fport, laddr, lport))
2236 return (inp);
2248 in_pcblookup_wild_match(const struct inpcb *inp, struct in_addr laddr,
2252 /* XXX inp locking */
2253 if ((inp->inp_vflag & INP_IPV4) == 0)
2256 if (inp->inp_faddr.s_addr != INADDR_ANY || inp->inp_lport != lport)
2258 if (inp->inp_laddr.s_addr == INADDR_ANY)
2260 if (inp->inp_laddr.s_addr == laddr.s_addr)
2272 struct inpcb *inp;
2279 CK_LIST_FOREACH(inp, head, inp_hash_wild) {
2282 match = in_pcblookup_wild_match(inp, laddr, lport);
2286 if (__predict_true(inp_smr_lock(inp, lockflags))) {
2287 match = in_pcblookup_wild_match(inp, laddr, lport);
2289 prison_check_ip4_locked(inp->inp_cred->cr_prison,
2291 return (inp);
2292 inp_unlock(inp, lockflags);
2309 struct inpcb *inp, *local_wild, *local_exact, *jail_wild;
2329 CK_LIST_FOREACH(inp, head, inp_hash_wild) {
2333 match = in_pcblookup_wild_match(inp, laddr, lport);
2337 injail = prison_flag(inp->inp_cred, PR_IP4) != 0;
2339 if (prison_check_ip4_locked(inp->inp_cred->cr_prison,
2349 return (inp);
2350 local_exact = inp;
2353 /* XXX inp locking, NULL check */
2354 if (inp->inp_vflag & INP_IPV6PROTO)
2355 local_wild_mapped = inp;
2359 jail_wild = inp;
2361 local_wild = inp;
2388 struct inpcb *inp;
2399 inp = in_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport);
2400 if (inp != NULL)
2401 return (inp);
2404 inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport,
2406 if (inp == NULL) {
2407 inp = in_pcblookup_hash_wild_locked(pcbinfo, laddr,
2412 return (inp);
2420 struct inpcb *inp;
2427 inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
2429 if (inp != NULL && !inp_trylock(inp, lockflags)) {
2430 in_pcbref(inp);
2432 inp_lock(inp, lockflags);
2433 if (in_pcbrele(inp, lockflags))
2435 inp = NULL;
2439 return (inp);
2447 struct inpcb *inp;
2457 inp = in_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport);
2458 if (inp != NULL) {
2459 if (__predict_true(inp_smr_lock(inp, lockflags))) {
2464 if (__predict_true(in_pcblookup_exact_match(inp,
2466 return (inp);
2467 inp_unlock(inp, lockflags);
2479 inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport,
2481 if (inp != NULL) {
2482 if (__predict_true(inp_smr_lock(inp, lockflags))) {
2483 if (__predict_true(in_pcblookup_wild_match(inp,
2485 return (inp);
2486 inp_unlock(inp, lockflags);
2488 inp = INP_LOOKUP_AGAIN;
2490 inp = in_pcblookup_hash_wild_smr(pcbinfo, laddr, lport,
2493 if (inp == INP_LOOKUP_AGAIN) {
2499 if (inp == NULL)
2502 return (inp);
2529 in_pcbjailed(const struct inpcb *inp, unsigned int flag)
2531 return (prison_flag(inp->inp_cred, flag) != 0);
2543 _in_pcbinshash_wild(struct inpcbhead *pcbhash, struct inpcb *inp)
2548 INP_LOCK_ASSERT(inp);
2549 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
2552 bound = inp->inp_laddr.s_addr != INADDR_ANY;
2553 if (!bound && (inp->inp_vflag & INP_IPV6PROTO) != 0) {
2556 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2560 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2564 injail = in_pcbjailed(inp, PR_IP4);
2570 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2576 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2584 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2590 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2592 CK_LIST_INSERT_BEFORE(last, inp, inp_hash_wild);
2600 _in6_pcbinshash_wild(struct inpcbhead *pcbhash, struct inpcb *inp)
2605 INP_LOCK_ASSERT(inp);
2606 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
2609 bound = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr);
2610 injail = in_pcbjailed(inp, PR_IP6);
2616 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2622 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2630 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2636 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2638 CK_LIST_INSERT_BEFORE(last, inp, inp_hash_wild);
2646 in_pcbinshash(struct inpcb *inp)
2650 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2655 INP_WLOCK_ASSERT(inp);
2657 KASSERT((inp->inp_flags & INP_INHASHLIST) == 0,
2661 if (inp->inp_vflag & INP_IPV6) {
2662 hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport,
2663 inp->inp_fport, pcbinfo->ipi_hashmask);
2664 connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr);
2668 hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport,
2669 inp->inp_fport, pcbinfo->ipi_hashmask);
2670 connected = !in_nullhost(inp->inp_faddr);
2679 INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)];
2685 if ((inp->inp_socket->so_options & SO_REUSEPORT_LB) != 0) {
2686 int error = in_pcbinslbgrouphash(inp, M_NODOM);
2695 if (phd->phd_port == inp->inp_lport)
2705 if ((inp->inp_flags & INP_INLBGROUP) != 0)
2706 in_pcbremlbgrouphash(inp);
2709 phd->phd_port = inp->inp_lport;
2713 inp->inp_phd = phd;
2714 CK_LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
2721 if (inp->inp_smr != SMR_SEQ_INVALID) {
2722 smr_wait(pcbinfo->ipi_smr, inp->inp_smr);
2723 inp->inp_smr = SMR_SEQ_INVALID;
2727 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_exact);
2730 if ((inp->inp_vflag & INP_IPV6) != 0)
2731 _in6_pcbinshash_wild(pcbhash, inp);
2734 _in_pcbinshash_wild(pcbhash, inp);
2736 inp->inp_flags |= INP_INHASHLIST;
2742 in_pcbremhash_locked(struct inpcb *inp)
2744 struct inpcbport *phd = inp->inp_phd;
2746 INP_WLOCK_ASSERT(inp);
2747 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
2748 MPASS(inp->inp_flags & INP_INHASHLIST);
2750 if ((inp->inp_flags & INP_INLBGROUP) != 0)
2751 in_pcbremlbgrouphash(inp);
2753 if (inp->inp_vflag & INP_IPV6) {
2754 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr))
2755 CK_LIST_REMOVE(inp, inp_hash_wild);
2757 CK_LIST_REMOVE(inp, inp_hash_exact);
2761 if (in_nullhost(inp->inp_faddr))
2762 CK_LIST_REMOVE(inp, inp_hash_wild);
2764 CK_LIST_REMOVE(inp, inp_hash_exact);
2766 CK_LIST_REMOVE(inp, inp_portlist);
2769 uma_zfree_smr(inp->inp_pcbinfo->ipi_portzone, phd);
2771 inp->inp_flags &= ~INP_INHASHLIST;
2775 in_pcbremhash(struct inpcb *inp)
2777 INP_HASH_WLOCK(inp->inp_pcbinfo);
2778 in_pcbremhash_locked(inp);
2779 INP_HASH_WUNLOCK(inp->inp_pcbinfo);
2789 in_pcbrehash(struct inpcb *inp)
2791 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2796 INP_WLOCK_ASSERT(inp);
2798 KASSERT(inp->inp_flags & INP_INHASHLIST,
2800 KASSERT(inp->inp_smr == SMR_SEQ_INVALID,
2801 ("%s: inp was disconnected", __func__));
2804 if (inp->inp_vflag & INP_IPV6) {
2805 hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport,
2806 inp->inp_fport, pcbinfo->ipi_hashmask);
2807 connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr);
2811 hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport,
2812 inp->inp_fport, pcbinfo->ipi_hashmask);
2813 connected = !in_nullhost(inp->inp_faddr);
2821 CK_LIST_REMOVE(inp, inp_hash_wild);
2823 CK_LIST_REMOVE(inp, inp_hash_exact);
2827 CK_LIST_INSERT_HEAD(head, inp, inp_hash_exact);
2830 CK_LIST_INSERT_HEAD(head, inp, inp_hash_wild);
2841 in_losing(struct inpcb *inp)
2844 RO_INVALIDATE_CACHE(&inp->inp_route);
2856 struct inpcb *inp;
2858 inp = sotoinpcb(so);
2859 KASSERT(inp != NULL, ("in_pcbsosetlabel: so->so_pcb == NULL"));
2861 INP_WLOCK(inp);
2863 mac_inpcb_sosetlabel(so, inp);
2865 INP_WUNLOCK(inp);
2870 inp_wlock(struct inpcb *inp)
2873 INP_WLOCK(inp);
2877 inp_wunlock(struct inpcb *inp)
2880 INP_WUNLOCK(inp);
2884 inp_rlock(struct inpcb *inp)
2887 INP_RLOCK(inp);
2891 inp_runlock(struct inpcb *inp)
2894 INP_RUNLOCK(inp);
2899 inp_lock_assert(struct inpcb *inp)
2902 INP_WLOCK_ASSERT(inp);
2906 inp_unlock_assert(struct inpcb *inp)
2909 INP_UNLOCK_ASSERT(inp);
2919 struct inpcb *inp;
2921 while ((inp = inp_next(&inpi)) != NULL)
2922 func(inp, arg);
2926 inp_inpcbtosocket(struct inpcb *inp)
2929 INP_WLOCK_ASSERT(inp);
2930 return (inp->inp_socket);
2934 inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp,
2938 INP_LOCK_ASSERT(inp);
2939 *laddr = inp->inp_laddr.s_addr;
2940 *faddr = inp->inp_faddr.s_addr;
2941 *lp = inp->inp_lport;
2942 *fp = inp->inp_fport;
2947 * the kernel-format in_pcb structure pointed to by inp. This is done to
2954 in_pcbtoxinpcb(const struct inpcb *inp, struct xinpcb *xi)
2959 if (inp->inp_socket)
2960 sotoxsocket(inp->inp_socket, &xi->xi_socket);
2961 bcopy(&inp->inp_inc, &xi->inp_inc, sizeof(struct in_conninfo));
2962 xi->inp_gencnt = inp->inp_gencnt;
2963 xi->inp_flow = inp->inp_flow;
2964 xi->inp_flowid = inp->inp_flowid;
2965 xi->inp_flowtype = inp->inp_flowtype;
2966 xi->inp_flags = inp->inp_flags;
2967 xi->inp_flags2 = inp->inp_flags2;
2968 xi->in6p_cksum = inp->in6p_cksum;
2969 xi->in6p_hops = inp->in6p_hops;
2970 xi->inp_ip_tos = inp->inp_ip_tos;
2971 xi->inp_vflag = inp->inp_vflag;
2972 xi->inp_ip_ttl = inp->inp_ip_ttl;
2973 xi->inp_ip_p = inp->inp_ip_p;
2974 xi->inp_ip_minttl = inp->inp_ip_minttl;
2984 struct inpcb *inp;
3035 while ((inp = inp_next(&inpi)) != NULL)
3036 if (inp->inp_gencnt == params->sop_id) {
3037 if (inp->inp_flags & INP_DROPPED) {
3038 INP_WUNLOCK(inp);
3041 so = inp->inp_socket;
3045 INP_WUNLOCK(inp);
3048 error = (*ctloutput_set)(inp, &sopt);
3052 if (inp == NULL)
3234 db_print_inpcb(struct inpcb *inp, const char *name, int indent)
3238 db_printf("%s at %p\n", name, inp);
3243 db_printf("inp_flow: 0x%x\n", inp->inp_flow);
3245 db_print_inconninfo(&inp->inp_inc, "inp_conninfo", indent);
3249 inp->inp_label, inp->inp_flags);
3250 db_print_inpflags(inp->inp_flags);
3254 db_printf("inp_sp: %p inp_vflag: 0x%x (", inp->inp_sp,
3255 inp->inp_vflag);
3256 db_print_inpvflag(inp->inp_vflag);
3261 inp->inp_ip_ttl, inp->inp_ip_p, inp->inp_ip_minttl);
3265 if (inp->inp_vflag & INP_IPV6) {
3267 "in6p_moptions: %p\n", inp->in6p_options,
3268 inp->in6p_outputopts, inp->in6p_moptions);
3270 "in6p_hops %u\n", inp->in6p_icmp6filt, inp->in6p_cksum,
3271 inp->in6p_hops);
3276 "inp_ip_moptions: %p\n", inp->inp_ip_tos,
3277 inp->inp_options, inp->inp_moptions);
3281 db_printf("inp_phd: %p inp_gencnt: %ju\n", inp->inp_phd,
3282 (uintmax_t)inp->inp_gencnt);
3287 struct inpcb *inp;
3293 inp = (struct inpcb *)addr;
3295 db_print_inpcb(inp, "inpcb", 0);
3301 * Modify TX rate limit based on the existing "inp->inp_snd_tag",
3305 in_pcbmodify_txrtlmt(struct inpcb *inp, uint32_t max_pacing_rate)
3314 mst = inp->inp_snd_tag;
3328 * "inp->inp_snd_tag", if any.
3331 in_pcbquery_txrtlmt(struct inpcb *inp, uint32_t *p_max_pacing_rate)
3337 mst = inp->inp_snd_tag;
3353 * "inp->inp_snd_tag", if any.
3356 in_pcbquery_txrlevel(struct inpcb *inp, uint32_t *p_txqueue_level)
3362 mst = inp->inp_snd_tag;
3377 * given by the "ifp" argument and save it in "inp->inp_snd_tag":
3380 in_pcbattach_txrtlmt(struct inpcb *inp, struct ifnet *ifp,
3389 .rate_limit.hdr.numa_domain = inp->inp_numa_domain,
3395 INP_WLOCK_ASSERT(inp);
3402 if (*st != NULL || (inp->inp_flags & INP_DROPPED) != 0)
3427 * Free an existing TX rate limit tag based on the "inp->inp_snd_tag",
3431 in_pcbdetach_txrtlmt(struct inpcb *inp)
3435 INP_WLOCK_ASSERT(inp);
3437 mst = inp->inp_snd_tag;
3438 inp->inp_snd_tag = NULL;
3450 in_pcboutput_txrtlmt_locked(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb, uint32_t max_pacing_rate)
3460 if (inp->inp_snd_tag != NULL && inp->inp_snd_tag->ifp != ifp) {
3461 in_pcbdetach_txrtlmt(inp);
3462 inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3472 if (max_pacing_rate == 0 && inp->inp_snd_tag == NULL) {
3475 if (inp->inp_snd_tag != NULL)
3476 in_pcbdetach_txrtlmt(inp);
3478 } else if (inp->inp_snd_tag == NULL) {
3487 error = in_pcbattach_txrtlmt(inp, ifp, M_HASHTYPE_GET(mb),
3488 mb->m_pkthdr.flowid, max_pacing_rate, &inp->inp_snd_tag);
3491 error = in_pcbmodify_txrtlmt(inp, max_pacing_rate);
3494 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
3505 in_pcboutput_txrtlmt(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb)
3511 if (inp == NULL)
3514 socket = inp->inp_socket;
3518 if (!INP_WLOCKED(inp)) {
3525 if (!INP_TRY_UPGRADE(inp))
3540 in_pcboutput_txrtlmt_locked(inp, ifp, mb, max_pacing_rate);
3543 INP_DOWNGRADE(inp);
3550 in_pcboutput_eagain(struct inpcb *inp)
3554 if (inp == NULL)
3557 if (inp->inp_snd_tag == NULL)
3560 if (!INP_WLOCKED(inp)) {
3567 if (!INP_TRY_UPGRADE(inp))
3575 in_pcbdetach_txrtlmt(inp);
3578 inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3581 INP_DOWNGRADE(inp);