Lines Matching +full:tcb +full:- +full:capture
2 /*-
3 * SPDX-License-Identifier: BSD-2-Clause
5 * Copyright (c) 2016-2018 Netflix, Inc.
147 "Logging mode for auto-selected sessions (default is TCP_LOG_STATE_TAIL)");
151 "Auto-select from all sessions (rather than just those with IDs)");
260 #define TCPID_BUCKET_LOCK_INIT(tlb) mtx_init(&((tlb)->tlb_mtx), "tcp log id bucket", NULL, MTX_DEF)
261 #define TCPID_BUCKET_LOCK_DESTROY(tlb) mtx_destroy(&((tlb)->tlb_mtx))
262 #define TCPID_BUCKET_LOCK(tlb) mtx_lock(&((tlb)->tlb_mtx))
263 #define TCPID_BUCKET_UNLOCK(tlb) mtx_unlock(&((tlb)->tlb_mtx))
264 #define TCPID_BUCKET_LOCK_ASSERT(tlb) mtx_assert(&((tlb)->tlb_mtx), MA_OWNED)
265 #define TCPID_BUCKET_UNLOCK_ASSERT(tlb) mtx_assert(&((tlb)->tlb_mtx), MA_NOTOWNED)
267 #define TCPID_BUCKET_REF(tlb) refcount_acquire(&((tlb)->tlb_refcnt))
268 #define TCPID_BUCKET_UNREF(tlb) refcount_release(&((tlb)->tlb_refcnt))
321 /* Do we want to select this session for auto-logging? */
327 * If we are doing auto-capturing, figure out whether we will capture in tcp_log_selectauto()
343 return strncmp(a->tlb_id, b->tlb_id, TCP_LOG_ID_LEN); in tcp_log_id_cmp()
375 KASSERT(SLIST_EMPTY(&tlb->tlb_head), in tcp_log_remove_bucket()
376 ("%s: Attempt to remove non-empty bucket", __func__)); in tcp_log_remove_bucket()
384 counter_u64_add(tcp_log_pcb_ids_cur, (int64_t)-1); in tcp_log_remove_bucket()
455 * The caller is responsible for freeing the tp->t_lin/tln node!
484 tlb = tp->t_lib; in tcp_log_remove_id_node()
485 tln = tp->t_lin; in tcp_log_remove_id_node()
498 SLIST_REMOVE(&tlb->tlb_head, tln, tcp_log_id_node, tln_list); in tcp_log_remove_id_node()
500 tp->t_lib = NULL; in tcp_log_remove_id_node()
501 tp->t_lin = NULL; in tcp_log_remove_id_node()
510 if (inp->inp_flags & INP_DROPPED) { \
527 if (V_tcp_perconn_stats_enable == 2 && tp->t_stats == NULL) in tcp_log_grow_tlb()
536 atomic_fetchadd_int(&tlb->tlb_reqcnt, 1); in tcp_log_increment_reqcnt()
549 tlb = tp->t_lib; in tcp_log_apply_ratio()
564 hash = hash32_buf(tlb->tlb_id, strlen(tlb->tlb_id), 0); in tcp_log_apply_ratio()
565 if (hash > ratio_hash_thresh && tp->_t_logstate == TCP_LOG_STATE_OFF && in tcp_log_apply_ratio()
566 tlb->tlb_logstate == TCP_LOG_STATE_OFF) { in tcp_log_apply_ratio()
573 tlb->tlb_logstate = TCP_LOG_STATE_RATIO_OFF; in tcp_log_apply_ratio()
612 tlb = tp->t_lib; in tcp_log_set_tag()
621 strlcpy(tlb->tlb_tag, tag, TCP_LOG_TAG_LEN); in tcp_log_set_tag()
658 same = ((tp->t_lib != NULL && !strcmp(tp->t_lib->tlb_id, id)) || in tcp_log_set_id()
659 (tp->t_lib == NULL && *id == 0)); in tcp_log_set_id()
660 if (tp->_t_logstate && STAILQ_FIRST(&tp->t_logs) && !same) { in tcp_log_set_id()
665 switch(tp->_t_logstate) { in tcp_log_set_id()
667 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from head at id switch", in tcp_log_set_id()
671 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from tail at id switch", in tcp_log_set_id()
675 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from continual at id switch", in tcp_log_set_id()
679 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from bbpoints at id switch", in tcp_log_set_id()
685 if (tp->t_lib != NULL) { in tcp_log_set_id()
686 tcp_log_increment_reqcnt(tp->t_lib); in tcp_log_set_id()
687 if ((tp->t_lib->tlb_logstate > TCP_LOG_STATE_OFF) && in tcp_log_set_id()
688 (tp->t_log_state_set == 0)) { in tcp_log_set_id()
691 tp->_t_logstate = tp->t_lib->tlb_logstate; in tcp_log_set_id()
693 if ((tp->t_lib->tlb_loglimit) && in tcp_log_set_id()
694 (tp->t_log_state_set == 0)) { in tcp_log_set_id()
697 tp->t_loglimit = tp->t_lib->tlb_loglimit; in tcp_log_set_id()
713 if (tp->t_lib != NULL) { in tcp_log_set_id()
714 tlb = tp->t_lib; in tcp_log_set_id()
749 * In cases 2-4, we will start over. In case 1, we will in tcp_log_set_id()
752 if (tlb == NULL || tp->t_lib != tlb) { in tcp_log_set_id()
777 tln = tp->t_lin; in tcp_log_set_id()
787 if (tp->t_lib != NULL || tp->t_lin != NULL) in tcp_log_set_id()
800 KASSERT(tp->t_lib == NULL, ("%s: tp->t_lib is not NULL", __func__)); in tcp_log_set_id()
815 tln->tln_inp = inp; in tcp_log_set_id()
816 tln->tln_tp = tp; in tcp_log_set_id()
837 KASSERT(tlb == NULL, ("%s:%d tlb unexpectedly non-NULL", in tcp_log_set_id()
845 * need to recheck some things when we re-acquire the lock. in tcp_log_set_id()
876 tlb->tlb_logstate = tcp_log_auto_mode; in tcp_log_set_id()
878 tlb->tlb_logstate = TCP_LOG_STATE_OFF; in tcp_log_set_id()
879 tlb->tlb_loglimit = 0; in tcp_log_set_id()
880 tlb->tlb_tag[0] = '\0'; /* Default to an empty tag. */ in tcp_log_set_id()
889 * can use strlcpy(). :-) in tcp_log_set_id()
891 strncpy(tlb->tlb_id, id, TCP_LOG_ID_LEN - 1); in tcp_log_set_id()
892 tlb->tlb_id[TCP_LOG_ID_LEN - 1] = '\0'; in tcp_log_set_id()
899 * been initialized. :-( in tcp_log_set_id()
901 SLIST_INIT(&tlb->tlb_head); in tcp_log_set_id()
902 refcount_init(&tlb->tlb_refcnt, 1); in tcp_log_set_id()
903 tlb->tlb_reqcnt = 1; in tcp_log_set_id()
904 memset(&tlb->tlb_mtx, 0, sizeof(struct mtx)); in tcp_log_set_id()
912 counter_u64_add(tcp_log_pcb_ids_cur, (int64_t)-1); \ in tcp_log_set_id()
913 counter_u64_add(tcp_log_pcb_ids_tot, (int64_t)-1); \ in tcp_log_set_id()
923 if (tp->t_lib != NULL) { in tcp_log_set_id()
960 if (tp->t_lib != NULL) { in tcp_log_set_id()
974 tcp_log_grow_tlb(tlb->tlb_id, tp); in tcp_log_set_id()
977 SLIST_INSERT_HEAD(&tlb->tlb_head, tln, tln_list); in tcp_log_set_id()
978 tp->t_lib = tlb; in tcp_log_set_id()
979 tp->t_lin = tln; in tcp_log_set_id()
980 if (tp->t_lib->tlb_logstate > TCP_LOG_STATE_OFF) { in tcp_log_set_id()
983 tp->_t_logstate = tp->t_lib->tlb_logstate; in tcp_log_set_id()
985 if (tp->t_lib->tlb_loglimit) { in tcp_log_set_id()
988 tp->t_loglimit = tp->t_lib->tlb_loglimit; in tcp_log_set_id()
1030 if (tp->t_lib != NULL) { in tcp_log_get_id()
1031 len = strlcpy(buf, tp->t_lib->tlb_id, TCP_LOG_ID_LEN); in tcp_log_get_id()
1033 ("%s:%d: tp->t_lib->tlb_id too long (%zu)", in tcp_log_get_id()
1059 tlb = tp->t_lib; in tcp_log_get_tag()
1065 len = strlcpy(buf, tlb->tlb_tag, TCP_LOG_TAG_LEN); in tcp_log_get_tag()
1067 ("%s:%d: tp->t_lib->tlb_tag too long (%zu)", in tcp_log_get_tag()
1099 return ((tp->t_lib == NULL) ? 0 : tp->t_lib->tlb_refcnt); in tcp_log_get_id_cnt()
1105 * entry. This should catch when we do a double-free/double-remove or
1106 * a double-add.
1114 refcnt = atomic_fetchadd_int(&log_entry->tlm_refcnt, 1); in _tcp_log_entry_refcnt_add()
1116 panic("%s:%d: log_entry(%p)->tlm_refcnt is %d (expected 0)", in _tcp_log_entry_refcnt_add()
1128 refcnt = atomic_fetchadd_int(&log_entry->tlm_refcnt, -1); in _tcp_log_entry_refcnt_rem()
1130 panic("%s:%d: log_entry(%p)->tlm_refcnt is %d (expected 1)", in _tcp_log_entry_refcnt_rem()
1153 (*count)--; in tcp_log_free_log_common()
1177 tp->t_lognum--; in tcp_log_remove_log_cleanup()
1178 KASSERT(tp->t_lognum >= 0, in tcp_log_remove_log_cleanup()
1179 ("%s: tp->t_lognum unexpectedly negative", __func__)); in tcp_log_remove_log_cleanup()
1187 KASSERT(log_entry == STAILQ_FIRST(&tp->t_logs), in tcp_log_remove_log_head()
1188 ("%s: attempt to remove non-HEAD log entry", __func__)); in tcp_log_remove_log_head()
1189 STAILQ_REMOVE_HEAD(&tp->t_logs, tlm_queue); in tcp_log_remove_log_head()
1207 tlm->tlm_refcnt = 0; in tcp_log_zone_init()
1222 if (tlm->tlm_refcnt != 0) in tcp_log_zone_ctor()
1223 panic("%s:%d: tlm(%p)->tlm_refcnt is %d (expected 0)", in tcp_log_zone_ctor()
1224 __func__, __LINE__, tlm, tlm->tlm_refcnt); in tcp_log_zone_ctor()
1236 if (tlm->tlm_refcnt != 0) in tcp_log_zone_dtor()
1237 panic("%s:%d: tlm(%p)->tlm_refcnt is %d (expected 0)", in tcp_log_zone_dtor()
1238 __func__, __LINE__, tlm, tlm->tlm_refcnt); in tcp_log_zone_dtor()
1280 /* Do per-TCPCB initialization. */
1285 /* A new TCPCB should start out zero-initialized. */ in tcp_log_tcpcbinit()
1286 STAILQ_INIT(&tp->t_logs); in tcp_log_tcpcbinit()
1289 * If we are doing auto-capturing, figure out whether we will capture in tcp_log_tcpcbinit()
1292 tp->t_loglimit = tcp_log_session_limit; in tcp_log_tcpcbinit()
1296 tp->_t_logstate = tcp_log_auto_mode; in tcp_log_tcpcbinit()
1297 tp->t_flags2 |= TF2_LOG_AUTO; in tcp_log_tcpcbinit()
1325 tln->tln_expiretime <= expiry_limit) { in tcp_log_expire()
1341 tln->tln_expiretime = SBT_MAX; in tcp_log_expire()
1347 tlb = tln->tln_bucket; in tcp_log_expire()
1359 INP_WLOCK(tln->tln_inp); in tcp_log_expire()
1360 if (!in_pcbrele_wlocked(tln->tln_inp)) in tcp_log_expire()
1361 INP_WUNLOCK(tln->tln_inp); in tcp_log_expire()
1364 tcp_log_free_entries(&tln->tln_entries, &tln->tln_count); in tcp_log_expire()
1380 * Get max(now + TCP_LOG_EXPIRE_INTVL, tln->tln_expiretime) and in tcp_log_expire()
1385 if (expiry_limit < tln->tln_expiretime) in tcp_log_expire()
1386 expiry_limit = tln->tln_expiretime; in tcp_log_expire()
1410 tln->tln_ie = inp->inp_inc.inc_ie; in tcp_log_move_tp_to_node()
1411 if (inp->inp_inc.inc_flags & INC_ISIPV6) in tcp_log_move_tp_to_node()
1412 tln->tln_af = AF_INET6; in tcp_log_move_tp_to_node()
1414 tln->tln_af = AF_INET; in tcp_log_move_tp_to_node()
1415 tln->tln_entries = tp->t_logs; in tcp_log_move_tp_to_node()
1416 tln->tln_count = tp->t_lognum; in tcp_log_move_tp_to_node()
1417 tln->tln_bucket = tp->t_lib; in tcp_log_move_tp_to_node()
1420 STAILQ_INIT(&tp->t_logs); in tcp_log_move_tp_to_node()
1421 tp->t_lognum = 0; in tcp_log_move_tp_to_node()
1424 /* Do per-TCPCB cleanup */
1434 if (tp->_t_logstate) { in tcp_log_tcpcbfini()
1442 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { in tcp_log_tcpcbfini()
1444 log.u_raw.u64_flex[i] = tp->tcp_cnt_counters[i]; in tcp_log_tcpcbfini()
1452 lgb->tlb_flex1 = TCP_NUM_CNT_COUNTERS; in tcp_log_tcpcbfini()
1453 lgb->tlb_flex2 = 1; in tcp_log_tcpcbfini()
1457 log.u_raw.u64_flex[i] = tp->tcp_proc_time[i]; in tcp_log_tcpcbfini()
1465 lgb->tlb_flex1 = TCP_NUM_CNT_COUNTERS; in tcp_log_tcpcbfini()
1466 lgb->tlb_flex2 = 2; in tcp_log_tcpcbfini()
1472 log.u_bbr.cur_del_rate = tp->t_end_info; in tcp_log_tcpcbfini()
1485 switch(tp->_t_logstate) { in tcp_log_tcpcbfini()
1487 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from head", in tcp_log_tcpcbfini()
1491 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from tail", in tcp_log_tcpcbfini()
1495 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from bbpoints", in tcp_log_tcpcbfini()
1499 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from continual", in tcp_log_tcpcbfini()
1505 * There are two ways we could keep logs: per-socket or per-ID. If in tcp_log_tcpcbfini()
1512 * this code. :-) in tcp_log_tcpcbfini()
1538 if (tp->t_lin != NULL) { in tcp_log_tcpcbfini()
1542 tln = tp->t_lin; in tcp_log_tcpcbfini()
1543 KASSERT(tln->tln_inp == inp, in tcp_log_tcpcbfini()
1544 ("%s: Mismatched inp (tln->tln_inp=%p, tp inpcb=%p)", in tcp_log_tcpcbfini()
1545 __func__, tln->tln_inp, inp)); in tcp_log_tcpcbfini()
1549 tp->t_lin = NULL; in tcp_log_tcpcbfini()
1550 tp->t_lib = NULL; in tcp_log_tcpcbfini()
1567 * via the expiry timer lets us avoid locking messy-ness here.) in tcp_log_tcpcbfini()
1569 tln->tln_expiretime = getsbinuptime(); in tcp_log_tcpcbfini()
1571 if (tln->tln_count) { in tcp_log_tcpcbfini()
1572 tln->tln_expiretime += TCP_LOG_EXPIRE_TIME; in tcp_log_tcpcbfini()
1581 tln->tln_expiretime, SBT_1S, tcp_log_expire, in tcp_log_tcpcbfini()
1587 callouttime = tln->tln_expiretime + in tcp_log_tcpcbfini()
1592 callouttime < tln_first->tln_expiretime) && in tcp_log_tcpcbfini()
1610 * mis-orderings as a bunch of "expire now" entries in tcp_log_tcpcbfini()
1616 tln->tln_expiretime < tln_first->tln_expiretime) in tcp_log_tcpcbfini()
1633 atomic_store_rel_int(&tln->tln_closed, 1); in tcp_log_tcpcbfini()
1636 while ((log_entry = STAILQ_FIRST(&tp->t_logs)) != NULL) in tcp_log_tcpcbfini()
1638 KASSERT(tp->t_lognum == 0, in tcp_log_tcpcbfini()
1639 ("%s: After freeing entries, tp->t_lognum=%d (expected 0)", in tcp_log_tcpcbfini()
1640 __func__, tp->t_lognum)); in tcp_log_tcpcbfini()
1645 * in a last-minute log). in tcp_log_tcpcbfini()
1647 tp->_t_logstate = TCP_LOG_STATE_OFF; in tcp_log_tcpcbfini()
1656 if (tp->t_lognum == 0) in tcp_log_purge_tp_logbuf()
1659 while ((log_entry = STAILQ_FIRST(&tp->t_logs)) != NULL) in tcp_log_purge_tp_logbuf()
1661 KASSERT(tp->t_lognum == 0, in tcp_log_purge_tp_logbuf()
1662 ("%s: After freeing entries, tp->t_lognum=%d (expected 0)", in tcp_log_purge_tp_logbuf()
1663 __func__, tp->t_lognum)); in tcp_log_purge_tp_logbuf()
1664 tp->_t_logstate = TCP_LOG_STATE_OFF; in tcp_log_purge_tp_logbuf()
1701 KASSERT(tp->_t_logstate == TCP_LOG_STATE_HEAD || in tcp_log_event()
1702 tp->_t_logstate == TCP_LOG_STATE_TAIL || in tcp_log_event()
1703 tp->_t_logstate == TCP_LOG_STATE_CONTINUAL || in tcp_log_event()
1704 tp->_t_logstate == TCP_LOG_STATE_HEAD_AUTO || in tcp_log_event()
1705 tp->_t_logstate == TCP_LOG_VIA_BBPOINTS || in tcp_log_event()
1706 tp->_t_logstate == TCP_LOG_STATE_TAIL_AUTO, in tcp_log_event()
1707 ("%s called with unexpected tp->_t_logstate (%d)", __func__, in tcp_log_event()
1708 tp->_t_logstate)); in tcp_log_event()
1715 logsn = tp->t_logsn++; in tcp_log_event()
1722 if (tp->t_lognum < tp->t_loglimit) { in tcp_log_event()
1724 tp->t_lognum++; in tcp_log_event()
1731 * Sacrifice auto-logged sessions without a log ID if in tcp_log_event()
1734 * or we are resource-constrained.) in tcp_log_event()
1736 if (tp->t_lib == NULL && (tp->t_flags2 & TF2_LOG_AUTO) && in tcp_log_event()
1744 tp->_t_logstate = TCP_LOG_STATE_OFF; in tcp_log_event()
1753 if (tp->_t_logstate == TCP_LOG_STATE_HEAD_AUTO && in tcp_log_event()
1754 !tcp_log_dump_tp_logbuf(tp, "auto-dumped from head", in tcp_log_event()
1756 tp->_t_logstate = TCP_LOG_STATE_OFF; in tcp_log_event()
1758 } else if ((tp->_t_logstate == TCP_LOG_STATE_CONTINUAL) && in tcp_log_event()
1759 !tcp_log_dump_tp_logbuf(tp, "auto-dumped from continual", in tcp_log_event()
1770 } else if ((tp->_t_logstate == TCP_LOG_VIA_BBPOINTS) && in tcp_log_event()
1771 !tcp_log_dump_tp_logbuf(tp, "auto-dumped from bbpoints", in tcp_log_event()
1781 } else if (tp->_t_logstate == TCP_LOG_STATE_HEAD_AUTO) in tcp_log_event()
1785 if (tp->_t_logstate == TCP_LOG_STATE_HEAD) { in tcp_log_event()
1786 tp->_t_logstate = TCP_LOG_STATE_OFF; in tcp_log_event()
1795 * here. Because we are re-using the buffer, the total in tcp_log_event()
1798 if ((log_entry = STAILQ_FIRST(&tp->t_logs)) == NULL) in tcp_log_event()
1800 STAILQ_REMOVE_HEAD(&tp->t_logs, tlm_queue); in tcp_log_event()
1808 log_buf = &log_entry->tlm_buf; in tcp_log_event()
1809 log_verbose = &log_entry->tlm_v; in tcp_log_event()
1813 microuptime(&log_buf->tlb_tv); in tcp_log_event()
1815 memcpy(&log_buf->tlb_tv, itv, sizeof(struct timeval)); in tcp_log_event()
1816 log_buf->tlb_ticks = ticks; in tcp_log_event()
1817 log_buf->tlb_sn = logsn; in tcp_log_event()
1818 log_buf->tlb_stackid = tp->t_fb->tfb_id; in tcp_log_event()
1819 log_buf->tlb_eventid = eventid; in tcp_log_event()
1820 log_buf->tlb_eventflags = 0; in tcp_log_event()
1821 log_buf->tlb_errno = errornum; in tcp_log_event()
1825 log_buf->tlb_eventflags |= TLB_FLAG_RXBUF; in tcp_log_event()
1826 log_buf->tlb_rxbuf.tls_sb_acc = rxbuf->sb_acc; in tcp_log_event()
1827 log_buf->tlb_rxbuf.tls_sb_ccc = rxbuf->sb_ccc; in tcp_log_event()
1828 log_buf->tlb_rxbuf.tls_sb_spare = 0; in tcp_log_event()
1830 log_buf->tlb_rxbuf.tls_sb_acc = 0; in tcp_log_event()
1831 log_buf->tlb_rxbuf.tls_sb_ccc = 0; in tcp_log_event()
1834 log_buf->tlb_eventflags |= TLB_FLAG_TXBUF; in tcp_log_event()
1835 log_buf->tlb_txbuf.tls_sb_acc = txbuf->sb_acc; in tcp_log_event()
1836 log_buf->tlb_txbuf.tls_sb_ccc = txbuf->sb_ccc; in tcp_log_event()
1837 log_buf->tlb_txbuf.tls_sb_spare = 0; in tcp_log_event()
1839 log_buf->tlb_txbuf.tls_sb_acc = 0; in tcp_log_event()
1840 log_buf->tlb_txbuf.tls_sb_ccc = 0; in tcp_log_event()
1843 #define COPY_STAT(f) log_buf->tlb_ ## f = tp->f in tcp_log_event()
1844 #define COPY_STAT_T(f) log_buf->tlb_ ## f = tp->t_ ## f in tcp_log_event()
1872 /* Copy stack-specific info. */ in tcp_log_event()
1874 memcpy(&log_buf->tlb_stackinfo, stackinfo, in tcp_log_event()
1875 sizeof(log_buf->tlb_stackinfo)); in tcp_log_event()
1876 log_buf->tlb_eventflags |= TLB_FLAG_STACKINFO; in tcp_log_event()
1880 log_buf->tlb_len = len; in tcp_log_event()
1884 log_buf->tlb_eventflags |= TLB_FLAG_HDR; in tcp_log_event()
1885 log_buf->tlb_th = *th; in tcp_log_event()
1887 tcp_fields_to_net(&log_buf->tlb_th); in tcp_log_event()
1888 optlen = (th->th_off << 2) - sizeof (struct tcphdr); in tcp_log_event()
1890 memcpy(log_buf->tlb_opts, th + 1, optlen); in tcp_log_event()
1892 memset(&log_buf->tlb_th, 0, sizeof(*th)); in tcp_log_event()
1897 log_buf->tlb_eventflags |= TLB_FLAG_VERBOSE; in tcp_log_event()
1899 strlcpy(log_verbose->tlv_snd_frm, output_caller, in tcp_log_event()
1902 *log_verbose->tlv_snd_frm = 0; in tcp_log_event()
1903 strlcpy(log_verbose->tlv_trace_func, func, TCP_FUNC_LEN); in tcp_log_event()
1904 log_verbose->tlv_trace_line = line; in tcp_log_event()
1908 STAILQ_INSERT_TAIL(&tp->t_logs, log_entry, tlm_queue); in tcp_log_event()
1927 while ((log_entry = STAILQ_FIRST(&tp->t_logs)) != NULL) in tcp_log_state_change()
1932 tp->_t_logstate = TCP_LOG_STATE_OFF; in tcp_log_state_change()
1946 if (tp->t_lib == NULL || in tcp_log_state_change()
1947 tp->t_lib->tlb_logstate != TCP_LOG_STATE_RATIO_OFF) { in tcp_log_state_change()
1948 tp->_t_logstate = state; in tcp_log_state_change()
1951 tp->_t_logstate = TCP_LOG_STATE_OFF; in tcp_log_state_change()
1960 tp->_t_logstate = TCP_LOG_STATE_OFF; in tcp_log_state_change()
1963 tp->t_flags2 &= ~(TF2_LOG_AUTO); in tcp_log_state_change()
1976 if ((target = tp->t_lognum / 2) == 0) in tcp_log_drain()
1990 * since I don't think the mbuf <-> BB log cleanup in tcp_log_drain()
1998 if (tp->_t_logstate == TCP_LOG_STATE_HEAD) { in tcp_log_drain()
1999 skip = tp->t_lognum - target; in tcp_log_drain()
2000 STAILQ_FOREACH(log_entry, &tp->t_logs, tlm_queue) in tcp_log_drain()
2001 if (!--skip) in tcp_log_drain()
2008 STAILQ_REMOVE_AFTER(&tp->t_logs, log_entry, tlm_queue); in tcp_log_drain()
2012 target--; in tcp_log_drain()
2018 } else if (tp->_t_logstate == TCP_LOG_STATE_HEAD_AUTO) { in tcp_log_drain()
2019 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from head at drain", in tcp_log_drain()
2021 } else if (tp->_t_logstate == TCP_LOG_STATE_TAIL_AUTO) { in tcp_log_drain()
2022 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from tail at drain", in tcp_log_drain()
2024 } else if (tp->_t_logstate == TCP_LOG_VIA_BBPOINTS) { in tcp_log_drain()
2025 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from bbpoints", in tcp_log_drain()
2027 } else if (tp->_t_logstate == TCP_LOG_STATE_CONTINUAL) { in tcp_log_drain()
2028 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from continual", in tcp_log_drain()
2031 while ((log_entry = STAILQ_FIRST(&tp->t_logs)) != NULL && in tcp_log_drain()
2032 target--) in tcp_log_drain()
2037 KASSERT(tp->t_lognum > 0, in tcp_log_drain()
2038 ("%s: After removing from head, tp->t_lognum was %d", in tcp_log_drain()
2050 if (sopt->sopt_td != NULL) in tcp_log_copyout()
2070 out_entry = (struct tcp_log_buffer *) sopt->sopt_val; in tcp_log_logs_to_buf()
2072 count--; in tcp_log_logs_to_buf()
2087 if (log_entry->tlm_buf.tlb_eventflags & TLB_FLAG_HDR) in tcp_log_logs_to_buf()
2091 error = tcp_log_copyout(sopt, &log_entry->tlm_buf, out_entry, in tcp_log_logs_to_buf()
2095 if (!(log_entry->tlm_buf.tlb_eventflags & TLB_FLAG_HDR)) { in tcp_log_logs_to_buf()
2098 sizeof(struct tcp_log_buffer) - entrysize); in tcp_log_logs_to_buf()
2105 if (log_entry->tlm_buf.tlb_eventflags & TLB_FLAG_VERBOSE) { in tcp_log_logs_to_buf()
2106 error = tcp_log_copyout(sopt, &log_entry->tlm_v, in tcp_log_logs_to_buf()
2107 out_entry->tlb_verbose, in tcp_log_logs_to_buf()
2154 if (sopt->sopt_valsize / (sizeof(struct tcp_log_buffer) + in tcp_log_getlogbuf()
2155 sizeof(struct tcp_log_verbose)) >= tp->t_lognum) { in tcp_log_getlogbuf()
2156 log_entry = STAILQ_LAST(&tp->t_logs, tcp_log_mem, tlm_queue); in tcp_log_getlogbuf()
2159 outnum = tp->t_lognum; in tcp_log_getlogbuf()
2164 STAILQ_FOREACH(log_next, &tp->t_logs, tlm_queue) { in tcp_log_getlogbuf()
2166 if (log_next->tlm_buf.tlb_eventflags & in tcp_log_getlogbuf()
2169 if ((sopt->sopt_valsize - outsize) < entrysize) in tcp_log_getlogbuf()
2175 KASSERT(outsize <= sopt->sopt_valsize, in tcp_log_getlogbuf()
2177 "space (%zu)", __func__, outsize, sopt->sopt_valsize)); in tcp_log_getlogbuf()
2183 * Copy traditional sooptcopyout() behavior: if sopt->sopt_val in tcp_log_getlogbuf()
2189 if (sopt->sopt_val == NULL) { in tcp_log_getlogbuf()
2195 if (sopt->sopt_valsize > outsize) in tcp_log_getlogbuf()
2196 sopt->sopt_valsize = outsize; in tcp_log_getlogbuf()
2211 * need to update the list book-keeping entries. in tcp_log_getlogbuf()
2215 KASSERT(outnum == tp->t_lognum, in tcp_log_getlogbuf()
2216 ("%s:%d: outnum (%d) should match tp->t_lognum (%d)", in tcp_log_getlogbuf()
2217 __func__, __LINE__, outnum, tp->t_lognum)); in tcp_log_getlogbuf()
2218 log_tailq = tp->t_logs; in tcp_log_getlogbuf()
2219 tp->t_lognum = 0; in tcp_log_getlogbuf()
2220 STAILQ_INIT(&tp->t_logs); in tcp_log_getlogbuf()
2223 KASSERT(outnum < tp->t_lognum, in tcp_log_getlogbuf()
2224 ("%s:%d: outnum (%d) not less than tp->t_lognum (%d)", in tcp_log_getlogbuf()
2225 __func__, __LINE__, outnum, tp->t_lognum)); in tcp_log_getlogbuf()
2226 STAILQ_FIRST(&log_tailq) = STAILQ_FIRST(&tp->t_logs); in tcp_log_getlogbuf()
2227 STAILQ_FIRST(&tp->t_logs) = STAILQ_NEXT(log_entry, tlm_queue); in tcp_log_getlogbuf()
2229 ("%s:%d: tp->t_logs is unexpectedly shorter than expected" in tcp_log_getlogbuf()
2230 "(tp: %p, log_tailq: %p, outnum: %d, tp->t_lognum: %d)", in tcp_log_getlogbuf()
2231 __func__, __LINE__, tp, &log_tailq, outnum, tp->t_lognum)); in tcp_log_getlogbuf()
2234 tp->t_lognum -= outnum; in tcp_log_getlogbuf()
2247 if ((inp->inp_flags & INP_DROPPED) == 0) { in tcp_log_getlogbuf()
2251 STAILQ_CONCAT(&log_tailq, &tp->t_logs); in tcp_log_getlogbuf()
2252 tp->t_logs = log_tailq; in tcp_log_getlogbuf()
2253 tp->t_lognum += outnum; in tcp_log_getlogbuf()
2258 KASSERT(((caddr_t)out_entry - (caddr_t)sopt->sopt_val) == in tcp_log_getlogbuf()
2261 (size_t)((caddr_t)out_entry - (caddr_t)sopt->sopt_val), in tcp_log_getlogbuf()
2271 sopt->sopt_valsize = (size_t)((caddr_t)out_entry - in tcp_log_getlogbuf()
2272 (caddr_t)sopt->sopt_val); in tcp_log_getlogbuf()
2288 tcp_log_free_entries(&entry->tldl_entries, &entry->tldl_count); in tcp_log_free_queue()
2291 if (entry->tldl_common.tldq_buf != NULL) in tcp_log_free_queue()
2292 free(entry->tldl_common.tldq_buf, M_TCPLOGDEV); in tcp_log_free_queue()
2309 /* Take a worst-case guess at space needs. */ in tcp_log_expandlogbuf()
2311 entry->tldl_count * (sizeof(struct tcp_log_buffer) + in tcp_log_expandlogbuf()
2316 counter_u64_add(tcp_log_que_fail5, entry->tldl_count); in tcp_log_expandlogbuf()
2321 sopt.sopt_valsize -= sizeof(struct tcp_log_header); in tcp_log_expandlogbuf()
2324 error = tcp_log_logs_to_buf(&sopt, &entry->tldl_entries, in tcp_log_expandlogbuf()
2325 (struct tcp_log_buffer **)&end, entry->tldl_count); in tcp_log_expandlogbuf()
2332 tcp_log_free_entries(&entry->tldl_entries, &entry->tldl_count); in tcp_log_expandlogbuf()
2333 entry->tldl_count = 0; in tcp_log_expandlogbuf()
2336 hdr->tlh_version = TCP_LOG_BUF_VER; in tcp_log_expandlogbuf()
2337 hdr->tlh_type = TCP_LOG_DEV_TYPE_BBR; in tcp_log_expandlogbuf()
2338 hdr->tlh_length = end - (uint8_t *)hdr; in tcp_log_expandlogbuf()
2339 hdr->tlh_ie = entry->tldl_ie; in tcp_log_expandlogbuf()
2340 hdr->tlh_af = entry->tldl_af; in tcp_log_expandlogbuf()
2341 getboottime(&hdr->tlh_offset); in tcp_log_expandlogbuf()
2342 strlcpy(hdr->tlh_id, entry->tldl_id, TCP_LOG_ID_LEN); in tcp_log_expandlogbuf()
2343 strlcpy(hdr->tlh_tag, entry->tldl_tag, TCP_LOG_TAG_LEN); in tcp_log_expandlogbuf()
2344 strlcpy(hdr->tlh_reason, entry->tldl_reason, TCP_LOG_REASON_LEN); in tcp_log_expandlogbuf()
2356 * If force is false, this will only dump auto-logged sessions if
2371 if (tp->t_lognum == 0) in tcp_log_dump_tp_logbuf()
2375 if (tp->t_lib == NULL && (tp->t_flags2 & TF2_LOG_AUTO) && in tcp_log_dump_tp_logbuf()
2385 while ((log_entry = STAILQ_FIRST(&tp->t_logs)) != NULL) in tcp_log_dump_tp_logbuf()
2387 KASSERT(tp->t_lognum == 0, in tcp_log_dump_tp_logbuf()
2388 ("%s: After freeing entries, tp->t_lognum=%d (expected 0)", in tcp_log_dump_tp_logbuf()
2389 __func__, tp->t_lognum)); in tcp_log_dump_tp_logbuf()
2390 tp->_t_logstate = TCP_LOG_STATE_OFF; in tcp_log_dump_tp_logbuf()
2413 * Note that this check is slightly overly-restrictive in in tcp_log_dump_tp_logbuf()
2414 * that the TCB can survive either of these events. in tcp_log_dump_tp_logbuf()
2420 if (inp->inp_flags & INP_DROPPED) { in tcp_log_dump_tp_logbuf()
2428 if (tp->t_lognum == 0) { in tcp_log_dump_tp_logbuf()
2435 if (tp->t_lib != NULL) { in tcp_log_dump_tp_logbuf()
2436 strlcpy(entry->tldl_id, tp->t_lib->tlb_id, TCP_LOG_ID_LEN); in tcp_log_dump_tp_logbuf()
2437 strlcpy(entry->tldl_tag, tp->t_lib->tlb_tag, TCP_LOG_TAG_LEN); in tcp_log_dump_tp_logbuf()
2439 strlcpy(entry->tldl_id, "UNKNOWN", TCP_LOG_ID_LEN); in tcp_log_dump_tp_logbuf()
2440 strlcpy(entry->tldl_tag, "UNKNOWN", TCP_LOG_TAG_LEN); in tcp_log_dump_tp_logbuf()
2443 strlcpy(entry->tldl_reason, reason, TCP_LOG_REASON_LEN); in tcp_log_dump_tp_logbuf()
2445 strlcpy(entry->tldl_reason, "UNKNOWN", TCP_LOG_REASON_LEN); in tcp_log_dump_tp_logbuf()
2446 entry->tldl_ie = inp->inp_inc.inc_ie; in tcp_log_dump_tp_logbuf()
2447 if (inp->inp_inc.inc_flags & INC_ISIPV6) in tcp_log_dump_tp_logbuf()
2448 entry->tldl_af = AF_INET6; in tcp_log_dump_tp_logbuf()
2450 entry->tldl_af = AF_INET; in tcp_log_dump_tp_logbuf()
2451 entry->tldl_entries = tp->t_logs; in tcp_log_dump_tp_logbuf()
2452 entry->tldl_count = tp->t_lognum; in tcp_log_dump_tp_logbuf()
2455 entry->tldl_common.tldq_buf = NULL; in tcp_log_dump_tp_logbuf()
2456 entry->tldl_common.tldq_xform = tcp_log_expandlogbuf; in tcp_log_dump_tp_logbuf()
2457 entry->tldl_common.tldq_dtor = tcp_log_free_queue; in tcp_log_dump_tp_logbuf()
2461 num_entries = tp->t_lognum; in tcp_log_dump_tp_logbuf()
2463 tp->t_lognum = 0; in tcp_log_dump_tp_logbuf()
2464 STAILQ_INIT(&tp->t_logs); in tcp_log_dump_tp_logbuf()
2495 tlb = tln->tln_bucket; in tcp_log_dump_node_logbuf()
2497 KASSERT(tlb->tlb_refcnt > 0, in tcp_log_dump_node_logbuf()
2500 KASSERT(tln->tln_closed, in tcp_log_dump_node_logbuf()
2505 if (tln->tln_count == 0) in tcp_log_dump_node_logbuf()
2525 entry->tldl_common.tldq_buf = NULL; in tcp_log_dump_node_logbuf()
2526 entry->tldl_common.tldq_xform = tcp_log_expandlogbuf; in tcp_log_dump_node_logbuf()
2527 entry->tldl_common.tldq_dtor = tcp_log_free_queue; in tcp_log_dump_node_logbuf()
2530 strlcpy(entry->tldl_id, tlb->tlb_id, TCP_LOG_ID_LEN); in tcp_log_dump_node_logbuf()
2531 strlcpy(entry->tldl_tag, tlb->tlb_tag, TCP_LOG_TAG_LEN); in tcp_log_dump_node_logbuf()
2533 strlcpy(entry->tldl_reason, reason, TCP_LOG_REASON_LEN); in tcp_log_dump_node_logbuf()
2535 strlcpy(entry->tldl_reason, "UNKNOWN", TCP_LOG_REASON_LEN); in tcp_log_dump_node_logbuf()
2536 entry->tldl_ie = tln->tln_ie; in tcp_log_dump_node_logbuf()
2537 entry->tldl_entries = tln->tln_entries; in tcp_log_dump_node_logbuf()
2538 entry->tldl_count = tln->tln_count; in tcp_log_dump_node_logbuf()
2539 entry->tldl_af = tln->tln_af; in tcp_log_dump_node_logbuf()
2586 SLIST_FOREACH_SAFE(cur_tln, &tlb->tlb_head, tln_list, tmp_tln) { in tcp_log_dumpbucketlogs()
2593 if (cur_tln->tln_closed) { in tcp_log_dumpbucketlogs()
2609 KASSERT(cur_tln->tln_expiretime > (sbintime_t) 0, in tcp_log_dumpbucketlogs()
2612 if (cur_tln->tln_expiretime == SBT_MAX) { in tcp_log_dumpbucketlogs()
2625 SLIST_REMOVE_HEAD(&tlb->tlb_head, tln_list); in tcp_log_dumpbucketlogs()
2629 * lock-ordering rules, we need to drop the expire in tcp_log_dumpbucketlogs()
2636 INP_WLOCK(cur_tln->tln_inp); in tcp_log_dumpbucketlogs()
2637 if (!in_pcbrele_wlocked(cur_tln->tln_inp)) in tcp_log_dumpbucketlogs()
2638 INP_WUNLOCK(cur_tln->tln_inp); in tcp_log_dumpbucketlogs()
2649 tcp_log_free_entries(&cur_tln->tln_entries, in tcp_log_dumpbucketlogs()
2650 &cur_tln->tln_count); in tcp_log_dumpbucketlogs()
2667 &cur_tln->tln_entries, in tcp_log_dumpbucketlogs()
2668 &cur_tln->tln_count); in tcp_log_dumpbucketlogs()
2696 /* Lock the INP and then re-check the state. */ in tcp_log_dumpbucketlogs()
2697 inp = cur_tln->tln_inp; in tcp_log_dumpbucketlogs()
2705 if (cur_tln->tln_closed) { in tcp_log_dumpbucketlogs()
2719 tp = cur_tln->tln_tp; in tcp_log_dumpbucketlogs()
2801 tlb = tp->t_lib; in tcp_log_dump_tp_bucket_logbufs()
2835 * stack-specific info to this trace event by overriding this
2841 if (tp->_t_logstate != TCP_LOG_STATE_OFF) { in tcp_log_flowend()
2843 TCP_LOG_EVENT(tp, NULL, &so->so_rcv, &so->so_snd, in tcp_log_flowend()
2863 if ((inp->inp_flags & INP_DROPPED) || in tcp_log_sendfile()
2864 (tp->_t_logstate == TCP_LOG_STATE_OFF)) { in tcp_log_sendfile()
2870 if (inp->inp_flags & INP_DROPPED) in tcp_log_sendfile()
2872 if (tp->_t_logstate != TCP_LOG_STATE_OFF) { in tcp_log_sendfile()
2882 &tptosocket(tp)->so_rcv, in tcp_log_sendfile()
2883 &tptosocket(tp)->so_snd, in tcp_log_sendfile()
2887 if (tp->t_tcpreq_req == 0) { in tcp_log_sendfile()
2892 if (tp->t_tcpreq_closed == 0) { in tcp_log_sendfile()
2898 ent = &tp->t_tcpreq_info[i]; in tcp_log_sendfile()
2899 if (ent->flags == TCP_TRK_TRACK_FLG_EMPTY) { in tcp_log_sendfile()
2903 if (ent->flags & TCP_TRK_TRACK_FLG_OPEN) { in tcp_log_sendfile()
2907 if (ent->flags & TCP_TRK_TRACK_FLG_COMP) { in tcp_log_sendfile()
2912 if ((ent->start == offset) || in tcp_log_sendfile()
2913 ((offset > ent->start) && (offset < ent->end))){ in tcp_log_sendfile()
2921 ent->flags |= TCP_TRK_TRACK_FLG_SEQV; in tcp_log_sendfile()
2926 * we must use sb_ccc since the data may still be in-flight in TLS. in tcp_log_sendfile()
2933 if (SEQ_GT((tp->snd_una + so->so_snd.sb_ccc), ent->end_seq)) in tcp_log_sendfile()
2934 ent->end_seq = tp->snd_una + so->so_snd.sb_ccc; in tcp_log_sendfile()
2935 if ((offset + nbytes) >= ent->end) { in tcp_log_sendfile()
2936 ent->flags |= TCP_TRK_TRACK_FLG_COMP; in tcp_log_sendfile()
2949 ent = &tp->t_tcpreq_info[i]; in tcp_log_sendfile()
2950 if (ent->flags == TCP_TRK_TRACK_FLG_EMPTY) { in tcp_log_sendfile()
2954 if ((ent->flags & TCP_TRK_TRACK_FLG_OPEN) == 0) in tcp_log_sendfile()
2957 if (ent->start == offset) { in tcp_log_sendfile()
2959 ent->start_seq = tp->snd_una + in tcp_log_sendfile()
2960 tptosocket(tp)->so_snd.sb_ccc; in tcp_log_sendfile()
2961 ent->flags |= TCP_TRK_TRACK_FLG_SEQV; in tcp_log_sendfile()
2963 } else if (offset > ent->start) { in tcp_log_sendfile()
2964 ent->flags |= TCP_TRK_TRACK_FLG_SEQV; in tcp_log_sendfile()