Lines Matching +full:d +full:- +full:tlb +full:- +full:size
2 /*-
3 * SPDX-License-Identifier: BSD-2-Clause
5 * Copyright (c) 2016-2018 Netflix, Inc.
147 "Logging mode for auto-selected sessions (default is TCP_LOG_STATE_TAIL)");
151 "Auto-select from all sessions (rather than just those with IDs)");
225 * D. To remove a node from the bucket, you must have that entry locked,
239 * d. Acquire a lock on the TCPID_BUCKET.
260 #define TCPID_BUCKET_LOCK_INIT(tlb) mtx_init(&((tlb)->tlb_mtx), "tcp log id bucket", NULL, MTX_DEF) argument
261 #define TCPID_BUCKET_LOCK_DESTROY(tlb) mtx_destroy(&((tlb)->tlb_mtx)) argument
262 #define TCPID_BUCKET_LOCK(tlb) mtx_lock(&((tlb)->tlb_mtx)) argument
263 #define TCPID_BUCKET_UNLOCK(tlb) mtx_unlock(&((tlb)->tlb_mtx)) argument
264 #define TCPID_BUCKET_LOCK_ASSERT(tlb) mtx_assert(&((tlb)->tlb_mtx), MA_OWNED) argument
265 #define TCPID_BUCKET_UNLOCK_ASSERT(tlb) mtx_assert(&((tlb)->tlb_mtx), MA_NOTOWNED) argument
267 #define TCPID_BUCKET_REF(tlb) refcount_acquire(&((tlb)->tlb_refcnt)) argument
268 #define TCPID_BUCKET_UNREF(tlb) refcount_release(&((tlb)->tlb_refcnt)) argument
321 /* Do we want to select this session for auto-logging? */
327 * If we are doing auto-capturing, figure out whether we will capture in tcp_log_selectauto()
343 return strncmp(a->tlb_id, b->tlb_id, TCP_LOG_ID_LEN); in tcp_log_id_cmp()
364 kassert_panic("%s:%d: unknown tree lock state", __func__, in RB_GENERATE_STATIC()
371 tcp_log_remove_bucket(struct tcp_log_id_bucket *tlb) in tcp_log_remove_bucket() argument
375 KASSERT(SLIST_EMPTY(&tlb->tlb_head), in tcp_log_remove_bucket()
376 ("%s: Attempt to remove non-empty bucket", __func__)); in tcp_log_remove_bucket()
377 if (RB_REMOVE(tcp_log_id_tree, &tcp_log_id_head, tlb) == NULL) { in tcp_log_remove_bucket()
379 kassert_panic("%s:%d: error removing element from tree", in tcp_log_remove_bucket()
383 TCPID_BUCKET_LOCK_DESTROY(tlb); in tcp_log_remove_bucket()
384 counter_u64_add(tcp_log_pcb_ids_cur, (int64_t)-1); in tcp_log_remove_bucket()
385 uma_zfree(tcp_log_id_bucket_zone, tlb); in tcp_log_remove_bucket()
391 * tlb: The bucket to unreference.
399 tcp_log_unref_bucket(struct tcp_log_id_bucket *tlb, int *tree_locked, in tcp_log_unref_bucket() argument
403 KASSERT(tlb != NULL, ("%s: called with NULL tlb", __func__)); in tcp_log_unref_bucket()
410 * Did we hold the last reference on the tlb? If so, we may need in tcp_log_unref_bucket()
415 while (TCPID_BUCKET_UNREF(tlb)) { in tcp_log_unref_bucket()
427 TCPID_BUCKET_REF(tlb); in tcp_log_unref_bucket()
430 TCPID_BUCKET_UNLOCK(tlb); in tcp_log_unref_bucket()
435 TCPID_BUCKET_LOCK(tlb); in tcp_log_unref_bucket()
445 tcp_log_remove_bucket(tlb); in tcp_log_unref_bucket()
455 * The caller is responsible for freeing the tp->t_lin/tln node!
457 * Note: one of tp or both tlb and tln must be supplied.
462 * tp: A pointer to the tcpcb. (optional; if specified, tlb and tln are ignored)
463 * tlb: A pointer to the bucket. (optional; ignored if tp is specified)
472 struct tcp_log_id_bucket *tlb, struct tcp_log_id_node *tln, in tcp_log_remove_id_node() argument
477 KASSERT(tp != NULL || (tlb != NULL && tln != NULL), in tcp_log_remove_id_node()
478 ("%s: called with tp=%p, tlb=%p, tln=%p", __func__, in tcp_log_remove_id_node()
479 tp, tlb, tln)); in tcp_log_remove_id_node()
484 tlb = tp->t_lib; in tcp_log_remove_id_node()
485 tln = tp->t_lin; in tcp_log_remove_id_node()
486 KASSERT(tlb != NULL, ("%s: unexpectedly NULL tlb", __func__)); in tcp_log_remove_id_node()
491 TCPID_BUCKET_LOCK_ASSERT(tlb); in tcp_log_remove_id_node()
498 SLIST_REMOVE(&tlb->tlb_head, tln, tcp_log_id_node, tln_list); in tcp_log_remove_id_node()
500 tp->t_lib = NULL; in tcp_log_remove_id_node()
501 tp->t_lin = NULL; in tcp_log_remove_id_node()
504 if (!tcp_log_unref_bucket(tlb, tree_locked, inp)) in tcp_log_remove_id_node()
505 TCPID_BUCKET_UNLOCK(tlb); in tcp_log_remove_id_node()
510 if (inp->inp_flags & INP_DROPPED) { \
527 if (V_tcp_perconn_stats_enable == 2 && tp->t_stats == NULL) in tcp_log_grow_tlb()
533 tcp_log_increment_reqcnt(struct tcp_log_id_bucket *tlb) in tcp_log_increment_reqcnt() argument
536 atomic_fetchadd_int(&tlb->tlb_reqcnt, 1); in tcp_log_increment_reqcnt()
542 struct tcp_log_id_bucket *tlb; in tcp_log_apply_ratio() local
549 tlb = tp->t_lib; in tcp_log_apply_ratio()
552 if (tlb == NULL) { in tcp_log_apply_ratio()
560 TCPID_BUCKET_REF(tlb); in tcp_log_apply_ratio()
562 TCPID_BUCKET_LOCK(tlb); in tcp_log_apply_ratio()
564 hash = hash32_buf(tlb->tlb_id, strlen(tlb->tlb_id), 0); in tcp_log_apply_ratio()
565 if (hash > ratio_hash_thresh && tp->_t_logstate == TCP_LOG_STATE_OFF && in tcp_log_apply_ratio()
566 tlb->tlb_logstate == TCP_LOG_STATE_OFF) { in tcp_log_apply_ratio()
573 tlb->tlb_logstate = TCP_LOG_STATE_RATIO_OFF; in tcp_log_apply_ratio()
582 if (!tcp_log_unref_bucket(tlb, &tree_locked, NULL)) in tcp_log_apply_ratio()
583 TCPID_BUCKET_UNLOCK(tlb); in tcp_log_apply_ratio()
606 struct tcp_log_id_bucket *tlb; in tcp_log_set_tag() local
612 tlb = tp->t_lib; in tcp_log_set_tag()
613 if (tlb == NULL) { in tcp_log_set_tag()
618 TCPID_BUCKET_REF(tlb); in tcp_log_set_tag()
620 TCPID_BUCKET_LOCK(tlb); in tcp_log_set_tag()
621 strlcpy(tlb->tlb_tag, tag, TCP_LOG_TAG_LEN); in tcp_log_set_tag()
622 if (!tcp_log_unref_bucket(tlb, &tree_locked, NULL)) in tcp_log_set_tag()
623 TCPID_BUCKET_UNLOCK(tlb); in tcp_log_set_tag()
644 struct tcp_log_id_bucket *tlb, *tmp_tlb; in tcp_log_set_id() local
650 tlb = NULL; in tcp_log_set_id()
658 same = ((tp->t_lib != NULL && !strcmp(tp->t_lib->tlb_id, id)) || in tcp_log_set_id()
659 (tp->t_lib == NULL && *id == 0)); in tcp_log_set_id()
660 if (tp->_t_logstate && STAILQ_FIRST(&tp->t_logs) && !same) { in tcp_log_set_id()
665 switch(tp->_t_logstate) { in tcp_log_set_id()
667 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from head at id switch", in tcp_log_set_id()
671 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from tail at id switch", in tcp_log_set_id()
675 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from continual at id switch", in tcp_log_set_id()
679 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from bbpoints at id switch", in tcp_log_set_id()
685 if (tp->t_lib != NULL) { in tcp_log_set_id()
686 tcp_log_increment_reqcnt(tp->t_lib); in tcp_log_set_id()
687 if ((tp->t_lib->tlb_logstate > TCP_LOG_STATE_OFF) && in tcp_log_set_id()
688 (tp->t_log_state_set == 0)) { in tcp_log_set_id()
691 tp->_t_logstate = tp->t_lib->tlb_logstate; in tcp_log_set_id()
693 if ((tp->t_lib->tlb_loglimit) && in tcp_log_set_id()
694 (tp->t_log_state_set == 0)) { in tcp_log_set_id()
697 tp->t_loglimit = tp->t_lib->tlb_loglimit; in tcp_log_set_id()
713 if (tp->t_lib != NULL) { in tcp_log_set_id()
714 tlb = tp->t_lib; in tcp_log_set_id()
715 TCPID_BUCKET_REF(tlb); in tcp_log_set_id()
722 TCPID_BUCKET_LOCK(tlb); in tcp_log_set_id()
730 if (tcp_log_unref_bucket(tlb, &tree_locked, inp)) { in tcp_log_set_id()
732 tlb = NULL; in tcp_log_set_id()
749 * In cases 2-4, we will start over. In case 1, we will in tcp_log_set_id()
752 if (tlb == NULL || tp->t_lib != tlb) { in tcp_log_set_id()
753 KASSERT(bucket_locked || tlb == NULL, in tcp_log_set_id()
754 ("%s: bucket_locked (%d) and tlb (%p) are " in tcp_log_set_id()
755 "inconsistent", __func__, bucket_locked, tlb)); in tcp_log_set_id()
758 TCPID_BUCKET_UNLOCK(tlb); in tcp_log_set_id()
760 tlb = NULL; in tcp_log_set_id()
771 * We will clear tlb and change the bucket_locked state just in tcp_log_set_id()
777 tln = tp->t_lin; in tcp_log_set_id()
778 tlb = NULL; in tcp_log_set_id()
787 if (tp->t_lib != NULL || tp->t_lin != NULL) in tcp_log_set_id()
800 KASSERT(tp->t_lib == NULL, ("%s: tp->t_lib is not NULL", __func__)); in tcp_log_set_id()
815 tln->tln_inp = inp; in tcp_log_set_id()
816 tln->tln_tp = tp; in tcp_log_set_id()
837 KASSERT(tlb == NULL, ("%s:%d tlb unexpectedly non-NULL", in tcp_log_set_id()
845 * need to recheck some things when we re-acquire the lock. in tcp_log_set_id()
864 tlb = uma_zalloc(tcp_log_id_bucket_zone, M_NOWAIT); in tcp_log_set_id()
865 if (tlb == NULL) { in tcp_log_set_id()
876 tlb->tlb_logstate = tcp_log_auto_mode; in tcp_log_set_id()
878 tlb->tlb_logstate = TCP_LOG_STATE_OFF; in tcp_log_set_id()
879 tlb->tlb_loglimit = 0; in tcp_log_set_id()
880 tlb->tlb_tag[0] = '\0'; /* Default to an empty tag. */ in tcp_log_set_id()
889 * can use strlcpy(). :-) in tcp_log_set_id()
891 strncpy(tlb->tlb_id, id, TCP_LOG_ID_LEN - 1); in tcp_log_set_id()
892 tlb->tlb_id[TCP_LOG_ID_LEN - 1] = '\0'; in tcp_log_set_id()
899 * been initialized. :-( in tcp_log_set_id()
901 SLIST_INIT(&tlb->tlb_head); in tcp_log_set_id()
902 refcount_init(&tlb->tlb_refcnt, 1); in tcp_log_set_id()
903 tlb->tlb_reqcnt = 1; in tcp_log_set_id()
904 memset(&tlb->tlb_mtx, 0, sizeof(struct mtx)); in tcp_log_set_id()
905 TCPID_BUCKET_LOCK_INIT(tlb); in tcp_log_set_id()
906 TCPID_BUCKET_LOCK(tlb); in tcp_log_set_id()
910 TCPID_BUCKET_LOCK_DESTROY(tlb); \ in tcp_log_set_id()
911 uma_zfree(tcp_log_id_bucket_zone, tlb); \ in tcp_log_set_id()
912 counter_u64_add(tcp_log_pcb_ids_cur, (int64_t)-1); \ in tcp_log_set_id()
913 counter_u64_add(tcp_log_pcb_ids_tot, (int64_t)-1); \ in tcp_log_set_id()
915 tlb = NULL; \ in tcp_log_set_id()
923 if (tp->t_lib != NULL) { in tcp_log_set_id()
930 tlb); in tcp_log_set_id()
933 "adding new bucket (%p)", __func__, tmp_tlb, tlb)); in tcp_log_set_id()
949 tlb = tmp_tlb; in tcp_log_set_id()
950 TCPID_BUCKET_LOCK(tlb); in tcp_log_set_id()
960 if (tp->t_lib != NULL) { in tcp_log_set_id()
961 TCPID_BUCKET_UNLOCK(tlb); in tcp_log_set_id()
963 tlb = NULL; in tcp_log_set_id()
968 TCPID_BUCKET_REF(tlb); in tcp_log_set_id()
971 tcp_log_increment_reqcnt(tlb); in tcp_log_set_id()
974 tcp_log_grow_tlb(tlb->tlb_id, tp); in tcp_log_set_id()
977 SLIST_INSERT_HEAD(&tlb->tlb_head, tln, tln_list); in tcp_log_set_id()
978 tp->t_lib = tlb; in tcp_log_set_id()
979 tp->t_lin = tln; in tcp_log_set_id()
980 if (tp->t_lib->tlb_logstate > TCP_LOG_STATE_OFF) { in tcp_log_set_id()
983 tp->_t_logstate = tp->t_lib->tlb_logstate; in tcp_log_set_id()
985 if (tp->t_lib->tlb_loglimit) { in tcp_log_set_id()
988 tp->t_loglimit = tp->t_lib->tlb_loglimit; in tcp_log_set_id()
1001 TCPID_BUCKET_LOCK_ASSERT(tlb); in tcp_log_set_id()
1002 TCPID_BUCKET_UNLOCK(tlb); in tcp_log_set_id()
1003 } else if (tlb != NULL) in tcp_log_set_id()
1004 TCPID_BUCKET_UNLOCK_ASSERT(tlb); in tcp_log_set_id()
1030 if (tp->t_lib != NULL) { in tcp_log_get_id()
1031 len = strlcpy(buf, tp->t_lib->tlb_id, TCP_LOG_ID_LEN); in tcp_log_get_id()
1033 ("%s:%d: tp->t_lib->tlb_id too long (%zu)", in tcp_log_get_id()
1052 struct tcp_log_id_bucket *tlb; in tcp_log_get_tag() local
1059 tlb = tp->t_lib; in tcp_log_get_tag()
1061 if (tlb != NULL) { in tcp_log_get_tag()
1062 TCPID_BUCKET_REF(tlb); in tcp_log_get_tag()
1064 TCPID_BUCKET_LOCK(tlb); in tcp_log_get_tag()
1065 len = strlcpy(buf, tlb->tlb_tag, TCP_LOG_TAG_LEN); in tcp_log_get_tag()
1067 ("%s:%d: tp->t_lib->tlb_tag too long (%zu)", in tcp_log_get_tag()
1069 if (!tcp_log_unref_bucket(tlb, &tree_locked, NULL)) in tcp_log_get_tag()
1070 TCPID_BUCKET_UNLOCK(tlb); in tcp_log_get_tag()
1099 return ((tp->t_lib == NULL) ? 0 : tp->t_lib->tlb_refcnt); in tcp_log_get_id_cnt()
1105 * entry. This should catch when we do a double-free/double-remove or
1106 * a double-add.
1114 refcnt = atomic_fetchadd_int(&log_entry->tlm_refcnt, 1); in _tcp_log_entry_refcnt_add()
1116 panic("%s:%d: log_entry(%p)->tlm_refcnt is %d (expected 0)", in _tcp_log_entry_refcnt_add()
1128 refcnt = atomic_fetchadd_int(&log_entry->tlm_refcnt, -1); in _tcp_log_entry_refcnt_rem()
1130 panic("%s:%d: log_entry(%p)->tlm_refcnt is %d (expected 1)", in _tcp_log_entry_refcnt_rem()
1153 (*count)--; in tcp_log_free_log_common()
1177 tp->t_lognum--; in tcp_log_remove_log_cleanup()
1178 KASSERT(tp->t_lognum >= 0, in tcp_log_remove_log_cleanup()
1179 ("%s: tp->t_lognum unexpectedly negative", __func__)); in tcp_log_remove_log_cleanup()
1187 KASSERT(log_entry == STAILQ_FIRST(&tp->t_logs), in tcp_log_remove_log_head()
1188 ("%s: attempt to remove non-HEAD log entry", __func__)); in tcp_log_remove_log_head()
1189 STAILQ_REMOVE_HEAD(&tp->t_logs, tlm_queue); in tcp_log_remove_log_head()
1200 tcp_log_zone_init(void *mem, int size, int flags __unused) in tcp_log_zone_init() argument
1204 KASSERT(size >= sizeof(struct tcp_log_mem), in tcp_log_zone_init()
1205 ("%s: unexpectedly short (%d) allocation", __func__, size)); in tcp_log_zone_init()
1207 tlm->tlm_refcnt = 0; in tcp_log_zone_init()
1215 tcp_log_zone_ctor(void *mem, int size, void *args __unused, int flags __unused) in tcp_log_zone_ctor() argument
1219 KASSERT(size >= sizeof(struct tcp_log_mem), in tcp_log_zone_ctor()
1220 ("%s: unexpectedly short (%d) allocation", __func__, size)); in tcp_log_zone_ctor()
1222 if (tlm->tlm_refcnt != 0) in tcp_log_zone_ctor()
1223 panic("%s:%d: tlm(%p)->tlm_refcnt is %d (expected 0)", in tcp_log_zone_ctor()
1224 __func__, __LINE__, tlm, tlm->tlm_refcnt); in tcp_log_zone_ctor()
1229 tcp_log_zone_dtor(void *mem, int size, void *args __unused) in tcp_log_zone_dtor() argument
1233 KASSERT(size >= sizeof(struct tcp_log_mem), in tcp_log_zone_dtor()
1234 ("%s: unexpectedly short (%d) allocation", __func__, size)); in tcp_log_zone_dtor()
1236 if (tlm->tlm_refcnt != 0) in tcp_log_zone_dtor()
1237 panic("%s:%d: tlm(%p)->tlm_refcnt is %d (expected 0)", in tcp_log_zone_dtor()
1238 __func__, __LINE__, tlm, tlm->tlm_refcnt); in tcp_log_zone_dtor()
1280 /* Do per-TCPCB initialization. */
1285 /* A new TCPCB should start out zero-initialized. */ in tcp_log_tcpcbinit()
1286 STAILQ_INIT(&tp->t_logs); in tcp_log_tcpcbinit()
1289 * If we are doing auto-capturing, figure out whether we will capture in tcp_log_tcpcbinit()
1292 tp->t_loglimit = tcp_log_session_limit; in tcp_log_tcpcbinit()
1296 tp->_t_logstate = tcp_log_auto_mode; in tcp_log_tcpcbinit()
1297 tp->t_flags2 |= TF2_LOG_AUTO; in tcp_log_tcpcbinit()
1305 struct tcp_log_id_bucket *tlb; in tcp_log_expire() local
1325 tln->tln_expiretime <= expiry_limit) { in tcp_log_expire()
1341 tln->tln_expiretime = SBT_MAX; in tcp_log_expire()
1347 tlb = tln->tln_bucket; in tcp_log_expire()
1348 TCPID_BUCKET_LOCK(tlb); in tcp_log_expire()
1349 if (tcp_log_remove_id_node(NULL, NULL, tlb, tln, &tree_locked)) { in tcp_log_expire()
1359 INP_WLOCK(tln->tln_inp); in tcp_log_expire()
1360 if (!in_pcbrele_wlocked(tln->tln_inp)) in tcp_log_expire()
1361 INP_WUNLOCK(tln->tln_inp); in tcp_log_expire()
1364 tcp_log_free_entries(&tln->tln_entries, &tln->tln_count); in tcp_log_expire()
1380 * Get max(now + TCP_LOG_EXPIRE_INTVL, tln->tln_expiretime) and in tcp_log_expire()
1385 if (expiry_limit < tln->tln_expiretime) in tcp_log_expire()
1386 expiry_limit = tln->tln_expiretime; in tcp_log_expire()
1410 tln->tln_ie = inp->inp_inc.inc_ie; in tcp_log_move_tp_to_node()
1411 if (inp->inp_inc.inc_flags & INC_ISIPV6) in tcp_log_move_tp_to_node()
1412 tln->tln_af = AF_INET6; in tcp_log_move_tp_to_node()
1414 tln->tln_af = AF_INET; in tcp_log_move_tp_to_node()
1415 tln->tln_entries = tp->t_logs; in tcp_log_move_tp_to_node()
1416 tln->tln_count = tp->t_lognum; in tcp_log_move_tp_to_node()
1417 tln->tln_bucket = tp->t_lib; in tcp_log_move_tp_to_node()
1420 STAILQ_INIT(&tp->t_logs); in tcp_log_move_tp_to_node()
1421 tp->t_lognum = 0; in tcp_log_move_tp_to_node()
1424 /* Do per-TCPCB cleanup */
1434 if (tp->_t_logstate) { in tcp_log_tcpcbfini()
1442 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { in tcp_log_tcpcbfini()
1444 log.u_raw.u64_flex[i] = tp->tcp_cnt_counters[i]; in tcp_log_tcpcbfini()
1452 lgb->tlb_flex1 = TCP_NUM_CNT_COUNTERS; in tcp_log_tcpcbfini()
1453 lgb->tlb_flex2 = 1; in tcp_log_tcpcbfini()
1457 log.u_raw.u64_flex[i] = tp->tcp_proc_time[i]; in tcp_log_tcpcbfini()
1465 lgb->tlb_flex1 = TCP_NUM_CNT_COUNTERS; in tcp_log_tcpcbfini()
1466 lgb->tlb_flex2 = 2; in tcp_log_tcpcbfini()
1472 log.u_bbr.cur_del_rate = tp->t_end_info; in tcp_log_tcpcbfini()
1485 switch(tp->_t_logstate) { in tcp_log_tcpcbfini()
1487 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from head", in tcp_log_tcpcbfini()
1491 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from tail", in tcp_log_tcpcbfini()
1495 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from bbpoints", in tcp_log_tcpcbfini()
1499 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from continual", in tcp_log_tcpcbfini()
1505 * There are two ways we could keep logs: per-socket or per-ID. If in tcp_log_tcpcbfini()
1512 * this code. :-) in tcp_log_tcpcbfini()
1516 * the bucket while we hold the lock (Rule D). Basically, no one can in tcp_log_tcpcbfini()
1538 if (tp->t_lin != NULL) { in tcp_log_tcpcbfini()
1542 tln = tp->t_lin; in tcp_log_tcpcbfini()
1543 KASSERT(tln->tln_inp == inp, in tcp_log_tcpcbfini()
1544 ("%s: Mismatched inp (tln->tln_inp=%p, tp inpcb=%p)", in tcp_log_tcpcbfini()
1545 __func__, tln->tln_inp, inp)); in tcp_log_tcpcbfini()
1549 tp->t_lin = NULL; in tcp_log_tcpcbfini()
1550 tp->t_lib = NULL; in tcp_log_tcpcbfini()
1567 * via the expiry timer lets us avoid locking messy-ness here.) in tcp_log_tcpcbfini()
1569 tln->tln_expiretime = getsbinuptime(); in tcp_log_tcpcbfini()
1571 if (tln->tln_count) { in tcp_log_tcpcbfini()
1572 tln->tln_expiretime += TCP_LOG_EXPIRE_TIME; in tcp_log_tcpcbfini()
1581 tln->tln_expiretime, SBT_1S, tcp_log_expire, in tcp_log_tcpcbfini()
1587 callouttime = tln->tln_expiretime + in tcp_log_tcpcbfini()
1592 callouttime < tln_first->tln_expiretime) && in tcp_log_tcpcbfini()
1610 * mis-orderings as a bunch of "expire now" entries in tcp_log_tcpcbfini()
1616 tln->tln_expiretime < tln_first->tln_expiretime) in tcp_log_tcpcbfini()
1633 atomic_store_rel_int(&tln->tln_closed, 1); in tcp_log_tcpcbfini()
1636 while ((log_entry = STAILQ_FIRST(&tp->t_logs)) != NULL) in tcp_log_tcpcbfini()
1638 KASSERT(tp->t_lognum == 0, in tcp_log_tcpcbfini()
1639 ("%s: After freeing entries, tp->t_lognum=%d (expected 0)", in tcp_log_tcpcbfini()
1640 __func__, tp->t_lognum)); in tcp_log_tcpcbfini()
1645 * in a last-minute log). in tcp_log_tcpcbfini()
1647 tp->_t_logstate = TCP_LOG_STATE_OFF; in tcp_log_tcpcbfini()
1656 if (tp->t_lognum == 0) in tcp_log_purge_tp_logbuf()
1659 while ((log_entry = STAILQ_FIRST(&tp->t_logs)) != NULL) in tcp_log_purge_tp_logbuf()
1661 KASSERT(tp->t_lognum == 0, in tcp_log_purge_tp_logbuf()
1662 ("%s: After freeing entries, tp->t_lognum=%d (expected 0)", in tcp_log_purge_tp_logbuf()
1663 __func__, tp->t_lognum)); in tcp_log_purge_tp_logbuf()
1664 tp->_t_logstate = TCP_LOG_STATE_OFF; in tcp_log_purge_tp_logbuf()
1686 ("%s called with inconsistent func (%p) and line (%d) arguments", in tcp_log_event()
1701 KASSERT(tp->_t_logstate == TCP_LOG_STATE_HEAD || in tcp_log_event()
1702 tp->_t_logstate == TCP_LOG_STATE_TAIL || in tcp_log_event()
1703 tp->_t_logstate == TCP_LOG_STATE_CONTINUAL || in tcp_log_event()
1704 tp->_t_logstate == TCP_LOG_STATE_HEAD_AUTO || in tcp_log_event()
1705 tp->_t_logstate == TCP_LOG_VIA_BBPOINTS || in tcp_log_event()
1706 tp->_t_logstate == TCP_LOG_STATE_TAIL_AUTO, in tcp_log_event()
1707 ("%s called with unexpected tp->_t_logstate (%d)", __func__, in tcp_log_event()
1708 tp->_t_logstate)); in tcp_log_event()
1715 logsn = tp->t_logsn++; in tcp_log_event()
1722 if (tp->t_lognum < tp->t_loglimit) { in tcp_log_event()
1724 tp->t_lognum++; in tcp_log_event()
1731 * Sacrifice auto-logged sessions without a log ID if in tcp_log_event()
1734 * or we are resource-constrained.) in tcp_log_event()
1736 if (tp->t_lib == NULL && (tp->t_flags2 & TF2_LOG_AUTO) && in tcp_log_event()
1740 panic("%s:%d: tcp_log_state_change() failed " in tcp_log_event()
1744 tp->_t_logstate = TCP_LOG_STATE_OFF; in tcp_log_event()
1753 if (tp->_t_logstate == TCP_LOG_STATE_HEAD_AUTO && in tcp_log_event()
1754 !tcp_log_dump_tp_logbuf(tp, "auto-dumped from head", in tcp_log_event()
1756 tp->_t_logstate = TCP_LOG_STATE_OFF; in tcp_log_event()
1758 } else if ((tp->_t_logstate == TCP_LOG_STATE_CONTINUAL) && in tcp_log_event()
1759 !tcp_log_dump_tp_logbuf(tp, "auto-dumped from continual", in tcp_log_event()
1770 } else if ((tp->_t_logstate == TCP_LOG_VIA_BBPOINTS) && in tcp_log_event()
1771 !tcp_log_dump_tp_logbuf(tp, "auto-dumped from bbpoints", in tcp_log_event()
1781 } else if (tp->_t_logstate == TCP_LOG_STATE_HEAD_AUTO) in tcp_log_event()
1785 if (tp->_t_logstate == TCP_LOG_STATE_HEAD) { in tcp_log_event()
1786 tp->_t_logstate = TCP_LOG_STATE_OFF; in tcp_log_event()
1795 * here. Because we are re-using the buffer, the total in tcp_log_event()
1798 if ((log_entry = STAILQ_FIRST(&tp->t_logs)) == NULL) in tcp_log_event()
1800 STAILQ_REMOVE_HEAD(&tp->t_logs, tlm_queue); in tcp_log_event()
1808 log_buf = &log_entry->tlm_buf; in tcp_log_event()
1809 log_verbose = &log_entry->tlm_v; in tcp_log_event()
1813 microuptime(&log_buf->tlb_tv); in tcp_log_event()
1815 memcpy(&log_buf->tlb_tv, itv, sizeof(struct timeval)); in tcp_log_event()
1816 log_buf->tlb_ticks = ticks; in tcp_log_event()
1817 log_buf->tlb_sn = logsn; in tcp_log_event()
1818 log_buf->tlb_stackid = tp->t_fb->tfb_id; in tcp_log_event()
1819 log_buf->tlb_eventid = eventid; in tcp_log_event()
1820 log_buf->tlb_eventflags = 0; in tcp_log_event()
1821 log_buf->tlb_errno = errornum; in tcp_log_event()
1825 log_buf->tlb_eventflags |= TLB_FLAG_RXBUF; in tcp_log_event()
1826 log_buf->tlb_rxbuf.tls_sb_acc = rxbuf->sb_acc; in tcp_log_event()
1827 log_buf->tlb_rxbuf.tls_sb_ccc = rxbuf->sb_ccc; in tcp_log_event()
1828 log_buf->tlb_rxbuf.tls_sb_spare = 0; in tcp_log_event()
1830 log_buf->tlb_rxbuf.tls_sb_acc = 0; in tcp_log_event()
1831 log_buf->tlb_rxbuf.tls_sb_ccc = 0; in tcp_log_event()
1834 log_buf->tlb_eventflags |= TLB_FLAG_TXBUF; in tcp_log_event()
1835 log_buf->tlb_txbuf.tls_sb_acc = txbuf->sb_acc; in tcp_log_event()
1836 log_buf->tlb_txbuf.tls_sb_ccc = txbuf->sb_ccc; in tcp_log_event()
1837 log_buf->tlb_txbuf.tls_sb_spare = 0; in tcp_log_event()
1839 log_buf->tlb_txbuf.tls_sb_acc = 0; in tcp_log_event()
1840 log_buf->tlb_txbuf.tls_sb_ccc = 0; in tcp_log_event()
1843 #define COPY_STAT(f) log_buf->tlb_ ## f = tp->f in tcp_log_event()
1844 #define COPY_STAT_T(f) log_buf->tlb_ ## f = tp->t_ ## f in tcp_log_event()
1872 /* Copy stack-specific info. */ in tcp_log_event()
1874 memcpy(&log_buf->tlb_stackinfo, stackinfo, in tcp_log_event()
1875 sizeof(log_buf->tlb_stackinfo)); in tcp_log_event()
1876 log_buf->tlb_eventflags |= TLB_FLAG_STACKINFO; in tcp_log_event()
1880 log_buf->tlb_len = len; in tcp_log_event()
1884 log_buf->tlb_eventflags |= TLB_FLAG_HDR; in tcp_log_event()
1885 log_buf->tlb_th = *th; in tcp_log_event()
1887 tcp_fields_to_net(&log_buf->tlb_th); in tcp_log_event()
1888 optlen = (th->th_off << 2) - sizeof (struct tcphdr); in tcp_log_event()
1890 memcpy(log_buf->tlb_opts, th + 1, optlen); in tcp_log_event()
1892 memset(&log_buf->tlb_th, 0, sizeof(*th)); in tcp_log_event()
1897 log_buf->tlb_eventflags |= TLB_FLAG_VERBOSE; in tcp_log_event()
1899 strlcpy(log_verbose->tlv_snd_frm, output_caller, in tcp_log_event()
1902 *log_verbose->tlv_snd_frm = 0; in tcp_log_event()
1903 strlcpy(log_verbose->tlv_trace_func, func, TCP_FUNC_LEN); in tcp_log_event()
1904 log_verbose->tlv_trace_line = line; in tcp_log_event()
1908 STAILQ_INSERT_TAIL(&tp->t_logs, log_entry, tlm_queue); in tcp_log_event()
1927 while ((log_entry = STAILQ_FIRST(&tp->t_logs)) != NULL) in tcp_log_state_change()
1932 tp->_t_logstate = TCP_LOG_STATE_OFF; in tcp_log_state_change()
1946 if (tp->t_lib == NULL || in tcp_log_state_change()
1947 tp->t_lib->tlb_logstate != TCP_LOG_STATE_RATIO_OFF) { in tcp_log_state_change()
1948 tp->_t_logstate = state; in tcp_log_state_change()
1951 tp->_t_logstate = TCP_LOG_STATE_OFF; in tcp_log_state_change()
1960 tp->_t_logstate = TCP_LOG_STATE_OFF; in tcp_log_state_change()
1963 tp->t_flags2 &= ~(TF2_LOG_AUTO); in tcp_log_state_change()
1976 if ((target = tp->t_lognum / 2) == 0) in tcp_log_drain()
1990 * since I don't think the mbuf <-> BB log cleanup in tcp_log_drain()
1998 if (tp->_t_logstate == TCP_LOG_STATE_HEAD) { in tcp_log_drain()
1999 skip = tp->t_lognum - target; in tcp_log_drain()
2000 STAILQ_FOREACH(log_entry, &tp->t_logs, tlm_queue) in tcp_log_drain()
2001 if (!--skip) in tcp_log_drain()
2008 STAILQ_REMOVE_AFTER(&tp->t_logs, log_entry, tlm_queue); in tcp_log_drain()
2012 target--; in tcp_log_drain()
2016 ("%s: After removing from tail, target was %d", __func__, in tcp_log_drain()
2018 } else if (tp->_t_logstate == TCP_LOG_STATE_HEAD_AUTO) { in tcp_log_drain()
2019 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from head at drain", in tcp_log_drain()
2021 } else if (tp->_t_logstate == TCP_LOG_STATE_TAIL_AUTO) { in tcp_log_drain()
2022 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from tail at drain", in tcp_log_drain()
2024 } else if (tp->_t_logstate == TCP_LOG_VIA_BBPOINTS) { in tcp_log_drain()
2025 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from bbpoints", in tcp_log_drain()
2027 } else if (tp->_t_logstate == TCP_LOG_STATE_CONTINUAL) { in tcp_log_drain()
2028 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from continual", in tcp_log_drain()
2031 while ((log_entry = STAILQ_FIRST(&tp->t_logs)) != NULL && in tcp_log_drain()
2032 target--) in tcp_log_drain()
2035 ("%s: After removing from head, target was %d", __func__, in tcp_log_drain()
2037 KASSERT(tp->t_lognum > 0, in tcp_log_drain()
2038 ("%s: After removing from head, tp->t_lognum was %d", in tcp_log_drain()
2050 if (sopt->sopt_td != NULL) in tcp_log_copyout()
2070 out_entry = (struct tcp_log_buffer *) sopt->sopt_val; in tcp_log_logs_to_buf()
2072 count--; in tcp_log_logs_to_buf()
2074 ("%s:%d: Exceeded expected count (%d) processing list %p", in tcp_log_logs_to_buf()
2087 if (log_entry->tlm_buf.tlb_eventflags & TLB_FLAG_HDR) in tcp_log_logs_to_buf()
2091 error = tcp_log_copyout(sopt, &log_entry->tlm_buf, out_entry, in tcp_log_logs_to_buf()
2095 if (!(log_entry->tlm_buf.tlb_eventflags & TLB_FLAG_HDR)) { in tcp_log_logs_to_buf()
2098 sizeof(struct tcp_log_buffer) - entrysize); in tcp_log_logs_to_buf()
2105 if (log_entry->tlm_buf.tlb_eventflags & TLB_FLAG_VERBOSE) { in tcp_log_logs_to_buf()
2106 error = tcp_log_copyout(sopt, &log_entry->tlm_v, in tcp_log_logs_to_buf()
2107 out_entry->tlb_verbose, in tcp_log_logs_to_buf()
2119 ("%s:%d: Less than expected count (%d) processing list %p" in tcp_log_logs_to_buf()
2120 " (%d remain)", __func__, __LINE__, orig_count, in tcp_log_logs_to_buf()
2150 * in the buffer. (However, get an exact size if we are using in tcp_log_getlogbuf()
2154 if (sopt->sopt_valsize / (sizeof(struct tcp_log_buffer) + in tcp_log_getlogbuf()
2155 sizeof(struct tcp_log_verbose)) >= tp->t_lognum) { in tcp_log_getlogbuf()
2156 log_entry = STAILQ_LAST(&tp->t_logs, tcp_log_mem, tlm_queue); in tcp_log_getlogbuf()
2159 outnum = tp->t_lognum; in tcp_log_getlogbuf()
2164 STAILQ_FOREACH(log_next, &tp->t_logs, tlm_queue) { in tcp_log_getlogbuf()
2166 if (log_next->tlm_buf.tlb_eventflags & in tcp_log_getlogbuf()
2169 if ((sopt->sopt_valsize - outsize) < entrysize) in tcp_log_getlogbuf()
2175 KASSERT(outsize <= sopt->sopt_valsize, in tcp_log_getlogbuf()
2176 ("%s: calculated output size (%zu) greater than available" in tcp_log_getlogbuf()
2177 "space (%zu)", __func__, outsize, sopt->sopt_valsize)); in tcp_log_getlogbuf()
2183 * Copy traditional sooptcopyout() behavior: if sopt->sopt_val in tcp_log_getlogbuf()
2187 * size they will need to get the log entries. in tcp_log_getlogbuf()
2189 if (sopt->sopt_val == NULL) { in tcp_log_getlogbuf()
2195 if (sopt->sopt_valsize > outsize) in tcp_log_getlogbuf()
2196 sopt->sopt_valsize = outsize; in tcp_log_getlogbuf()
2211 * need to update the list book-keeping entries. in tcp_log_getlogbuf()
2215 KASSERT(outnum == tp->t_lognum, in tcp_log_getlogbuf()
2216 ("%s:%d: outnum (%d) should match tp->t_lognum (%d)", in tcp_log_getlogbuf()
2217 __func__, __LINE__, outnum, tp->t_lognum)); in tcp_log_getlogbuf()
2218 log_tailq = tp->t_logs; in tcp_log_getlogbuf()
2219 tp->t_lognum = 0; in tcp_log_getlogbuf()
2220 STAILQ_INIT(&tp->t_logs); in tcp_log_getlogbuf()
2223 KASSERT(outnum < tp->t_lognum, in tcp_log_getlogbuf()
2224 ("%s:%d: outnum (%d) not less than tp->t_lognum (%d)", in tcp_log_getlogbuf()
2225 __func__, __LINE__, outnum, tp->t_lognum)); in tcp_log_getlogbuf()
2226 STAILQ_FIRST(&log_tailq) = STAILQ_FIRST(&tp->t_logs); in tcp_log_getlogbuf()
2227 STAILQ_FIRST(&tp->t_logs) = STAILQ_NEXT(log_entry, tlm_queue); in tcp_log_getlogbuf()
2229 ("%s:%d: tp->t_logs is unexpectedly shorter than expected" in tcp_log_getlogbuf()
2230 "(tp: %p, log_tailq: %p, outnum: %d, tp->t_lognum: %d)", in tcp_log_getlogbuf()
2231 __func__, __LINE__, tp, &log_tailq, outnum, tp->t_lognum)); in tcp_log_getlogbuf()
2234 tp->t_lognum -= outnum; in tcp_log_getlogbuf()
2247 if ((inp->inp_flags & INP_DROPPED) == 0) { in tcp_log_getlogbuf()
2251 STAILQ_CONCAT(&log_tailq, &tp->t_logs); in tcp_log_getlogbuf()
2252 tp->t_logs = log_tailq; in tcp_log_getlogbuf()
2253 tp->t_lognum += outnum; in tcp_log_getlogbuf()
2258 KASSERT(((caddr_t)out_entry - (caddr_t)sopt->sopt_val) == in tcp_log_getlogbuf()
2259 outsize, ("%s: Actual output size (%zu) != " in tcp_log_getlogbuf()
2260 "calculated output size (%zu)", __func__, in tcp_log_getlogbuf()
2261 (size_t)((caddr_t)out_entry - (caddr_t)sopt->sopt_val), in tcp_log_getlogbuf()
2271 sopt->sopt_valsize = (size_t)((caddr_t)out_entry - in tcp_log_getlogbuf()
2272 (caddr_t)sopt->sopt_val); in tcp_log_getlogbuf()
2288 tcp_log_free_entries(&entry->tldl_entries, &entry->tldl_count); in tcp_log_free_queue()
2291 if (entry->tldl_common.tldq_buf != NULL) in tcp_log_free_queue()
2292 free(entry->tldl_common.tldq_buf, M_TCPLOGDEV); in tcp_log_free_queue()
2309 /* Take a worst-case guess at space needs. */ in tcp_log_expandlogbuf()
2311 entry->tldl_count * (sizeof(struct tcp_log_buffer) + in tcp_log_expandlogbuf()
2316 counter_u64_add(tcp_log_que_fail5, entry->tldl_count); in tcp_log_expandlogbuf()
2321 sopt.sopt_valsize -= sizeof(struct tcp_log_header); in tcp_log_expandlogbuf()
2324 error = tcp_log_logs_to_buf(&sopt, &entry->tldl_entries, in tcp_log_expandlogbuf()
2325 (struct tcp_log_buffer **)&end, entry->tldl_count); in tcp_log_expandlogbuf()
2332 tcp_log_free_entries(&entry->tldl_entries, &entry->tldl_count); in tcp_log_expandlogbuf()
2333 entry->tldl_count = 0; in tcp_log_expandlogbuf()
2336 hdr->tlh_version = TCP_LOG_BUF_VER; in tcp_log_expandlogbuf()
2337 hdr->tlh_type = TCP_LOG_DEV_TYPE_BBR; in tcp_log_expandlogbuf()
2338 hdr->tlh_length = end - (uint8_t *)hdr; in tcp_log_expandlogbuf()
2339 hdr->tlh_ie = entry->tldl_ie; in tcp_log_expandlogbuf()
2340 hdr->tlh_af = entry->tldl_af; in tcp_log_expandlogbuf()
2341 getboottime(&hdr->tlh_offset); in tcp_log_expandlogbuf()
2342 strlcpy(hdr->tlh_id, entry->tldl_id, TCP_LOG_ID_LEN); in tcp_log_expandlogbuf()
2343 strlcpy(hdr->tlh_tag, entry->tldl_tag, TCP_LOG_TAG_LEN); in tcp_log_expandlogbuf()
2344 strlcpy(hdr->tlh_reason, entry->tldl_reason, TCP_LOG_REASON_LEN); in tcp_log_expandlogbuf()
2356 * If force is false, this will only dump auto-logged sessions if
2371 if (tp->t_lognum == 0) in tcp_log_dump_tp_logbuf()
2375 if (tp->t_lib == NULL && (tp->t_flags2 & TF2_LOG_AUTO) && in tcp_log_dump_tp_logbuf()
2385 while ((log_entry = STAILQ_FIRST(&tp->t_logs)) != NULL) in tcp_log_dump_tp_logbuf()
2387 KASSERT(tp->t_lognum == 0, in tcp_log_dump_tp_logbuf()
2388 ("%s: After freeing entries, tp->t_lognum=%d (expected 0)", in tcp_log_dump_tp_logbuf()
2389 __func__, tp->t_lognum)); in tcp_log_dump_tp_logbuf()
2390 tp->_t_logstate = TCP_LOG_STATE_OFF; in tcp_log_dump_tp_logbuf()
2413 * Note that this check is slightly overly-restrictive in in tcp_log_dump_tp_logbuf()
2420 if (inp->inp_flags & INP_DROPPED) { in tcp_log_dump_tp_logbuf()
2428 if (tp->t_lognum == 0) { in tcp_log_dump_tp_logbuf()
2435 if (tp->t_lib != NULL) { in tcp_log_dump_tp_logbuf()
2436 strlcpy(entry->tldl_id, tp->t_lib->tlb_id, TCP_LOG_ID_LEN); in tcp_log_dump_tp_logbuf()
2437 strlcpy(entry->tldl_tag, tp->t_lib->tlb_tag, TCP_LOG_TAG_LEN); in tcp_log_dump_tp_logbuf()
2439 strlcpy(entry->tldl_id, "UNKNOWN", TCP_LOG_ID_LEN); in tcp_log_dump_tp_logbuf()
2440 strlcpy(entry->tldl_tag, "UNKNOWN", TCP_LOG_TAG_LEN); in tcp_log_dump_tp_logbuf()
2443 strlcpy(entry->tldl_reason, reason, TCP_LOG_REASON_LEN); in tcp_log_dump_tp_logbuf()
2445 strlcpy(entry->tldl_reason, "UNKNOWN", TCP_LOG_REASON_LEN); in tcp_log_dump_tp_logbuf()
2446 entry->tldl_ie = inp->inp_inc.inc_ie; in tcp_log_dump_tp_logbuf()
2447 if (inp->inp_inc.inc_flags & INC_ISIPV6) in tcp_log_dump_tp_logbuf()
2448 entry->tldl_af = AF_INET6; in tcp_log_dump_tp_logbuf()
2450 entry->tldl_af = AF_INET; in tcp_log_dump_tp_logbuf()
2451 entry->tldl_entries = tp->t_logs; in tcp_log_dump_tp_logbuf()
2452 entry->tldl_count = tp->t_lognum; in tcp_log_dump_tp_logbuf()
2455 entry->tldl_common.tldq_buf = NULL; in tcp_log_dump_tp_logbuf()
2456 entry->tldl_common.tldq_xform = tcp_log_expandlogbuf; in tcp_log_dump_tp_logbuf()
2457 entry->tldl_common.tldq_dtor = tcp_log_free_queue; in tcp_log_dump_tp_logbuf()
2461 num_entries = tp->t_lognum; in tcp_log_dump_tp_logbuf()
2463 tp->t_lognum = 0; in tcp_log_dump_tp_logbuf()
2464 STAILQ_INIT(&tp->t_logs); in tcp_log_dump_tp_logbuf()
2493 struct tcp_log_id_bucket *tlb; in tcp_log_dump_node_logbuf() local
2495 tlb = tln->tln_bucket; in tcp_log_dump_node_logbuf()
2496 TCPID_BUCKET_LOCK_ASSERT(tlb); in tcp_log_dump_node_logbuf()
2497 KASSERT(tlb->tlb_refcnt > 0, in tcp_log_dump_node_logbuf()
2498 ("%s:%d: Called with unreferenced bucket (tln=%p, tlb=%p)", in tcp_log_dump_node_logbuf()
2499 __func__, __LINE__, tln, tlb)); in tcp_log_dump_node_logbuf()
2500 KASSERT(tln->tln_closed, in tcp_log_dump_node_logbuf()
2501 ("%s:%d: Called for node with tln_closed==false (tln=%p)", in tcp_log_dump_node_logbuf()
2505 if (tln->tln_count == 0) in tcp_log_dump_node_logbuf()
2518 TCPID_BUCKET_UNLOCK(tlb); in tcp_log_dump_node_logbuf()
2521 TCPID_BUCKET_LOCK(tlb); in tcp_log_dump_node_logbuf()
2525 entry->tldl_common.tldq_buf = NULL; in tcp_log_dump_node_logbuf()
2526 entry->tldl_common.tldq_xform = tcp_log_expandlogbuf; in tcp_log_dump_node_logbuf()
2527 entry->tldl_common.tldq_dtor = tcp_log_free_queue; in tcp_log_dump_node_logbuf()
2530 strlcpy(entry->tldl_id, tlb->tlb_id, TCP_LOG_ID_LEN); in tcp_log_dump_node_logbuf()
2531 strlcpy(entry->tldl_tag, tlb->tlb_tag, TCP_LOG_TAG_LEN); in tcp_log_dump_node_logbuf()
2533 strlcpy(entry->tldl_reason, reason, TCP_LOG_REASON_LEN); in tcp_log_dump_node_logbuf()
2535 strlcpy(entry->tldl_reason, "UNKNOWN", TCP_LOG_REASON_LEN); in tcp_log_dump_node_logbuf()
2536 entry->tldl_ie = tln->tln_ie; in tcp_log_dump_node_logbuf()
2537 entry->tldl_entries = tln->tln_entries; in tcp_log_dump_node_logbuf()
2538 entry->tldl_count = tln->tln_count; in tcp_log_dump_node_logbuf()
2539 entry->tldl_af = tln->tln_af; in tcp_log_dump_node_logbuf()
2557 tcp_log_dumpbucketlogs(struct tcp_log_id_bucket *tlb, char *reason) in tcp_log_dumpbucketlogs() argument
2566 TCPID_BUCKET_LOCK_ASSERT(tlb); in tcp_log_dumpbucketlogs()
2572 TCPID_BUCKET_REF(tlb); in tcp_log_dumpbucketlogs()
2586 SLIST_FOREACH_SAFE(cur_tln, &tlb->tlb_head, tln_list, tmp_tln) { in tcp_log_dumpbucketlogs()
2593 if (cur_tln->tln_closed) { in tcp_log_dumpbucketlogs()
2609 KASSERT(cur_tln->tln_expiretime > (sbintime_t) 0, in tcp_log_dumpbucketlogs()
2610 ("%s:%d: node on the expire queue without positive " in tcp_log_dumpbucketlogs()
2612 if (cur_tln->tln_expiretime == SBT_MAX) { in tcp_log_dumpbucketlogs()
2625 SLIST_REMOVE_HEAD(&tlb->tlb_head, tln_list); in tcp_log_dumpbucketlogs()
2629 * lock-ordering rules, we need to drop the expire in tcp_log_dumpbucketlogs()
2636 INP_WLOCK(cur_tln->tln_inp); in tcp_log_dumpbucketlogs()
2637 if (!in_pcbrele_wlocked(cur_tln->tln_inp)) in tcp_log_dumpbucketlogs()
2638 INP_WUNLOCK(cur_tln->tln_inp); in tcp_log_dumpbucketlogs()
2640 if (tcp_log_unref_bucket(tlb, &tree_locked, NULL)) { in tcp_log_dumpbucketlogs()
2649 tcp_log_free_entries(&cur_tln->tln_entries, in tcp_log_dumpbucketlogs()
2650 &cur_tln->tln_count); in tcp_log_dumpbucketlogs()
2667 &cur_tln->tln_entries, in tcp_log_dumpbucketlogs()
2668 &cur_tln->tln_count); in tcp_log_dumpbucketlogs()
2677 * (which tracks the previous entry still on the tlb in tcp_log_dumpbucketlogs()
2696 /* Lock the INP and then re-check the state. */ in tcp_log_dumpbucketlogs()
2697 inp = cur_tln->tln_inp; in tcp_log_dumpbucketlogs()
2705 if (cur_tln->tln_closed) { in tcp_log_dumpbucketlogs()
2719 tp = cur_tln->tln_tp; in tcp_log_dumpbucketlogs()
2726 tlb, ("%s: %d: bucket mismatch for node %p", in tcp_log_dumpbucketlogs()
2742 ("%s: %d: tree unexpectedly locked", __func__, __LINE__)); in tcp_log_dumpbucketlogs()
2767 if (!tcp_log_unref_bucket(tlb, &tree_locked, NULL)) in tcp_log_dumpbucketlogs()
2768 TCPID_BUCKET_UNLOCK(tlb); in tcp_log_dumpbucketlogs()
2796 struct tcp_log_id_bucket *tlb; in tcp_log_dump_tp_bucket_logbufs() local
2801 tlb = tp->t_lib; in tcp_log_dump_tp_bucket_logbufs()
2802 if (tlb == NULL) { in tcp_log_dump_tp_bucket_logbufs()
2811 TCPID_BUCKET_REF(tlb); in tcp_log_dump_tp_bucket_logbufs()
2813 TCPID_BUCKET_LOCK(tlb); in tcp_log_dump_tp_bucket_logbufs()
2817 if (tcp_log_unref_bucket(tlb, &tree_locked, NULL)) { in tcp_log_dump_tp_bucket_logbufs()
2830 tcp_log_dumpbucketlogs(tlb, reason); in tcp_log_dump_tp_bucket_logbufs()
2835 * stack-specific info to this trace event by overriding this
2841 if (tp->_t_logstate != TCP_LOG_STATE_OFF) { in tcp_log_flowend()
2843 TCP_LOG_EVENT(tp, NULL, &so->so_rcv, &so->so_snd, in tcp_log_flowend()
2863 if ((inp->inp_flags & INP_DROPPED) || in tcp_log_sendfile()
2864 (tp->_t_logstate == TCP_LOG_STATE_OFF)) { in tcp_log_sendfile()
2870 if (inp->inp_flags & INP_DROPPED) in tcp_log_sendfile()
2872 if (tp->_t_logstate != TCP_LOG_STATE_OFF) { in tcp_log_sendfile()
2882 &tptosocket(tp)->so_rcv, in tcp_log_sendfile()
2883 &tptosocket(tp)->so_snd, in tcp_log_sendfile()
2887 if (tp->t_tcpreq_req == 0) { in tcp_log_sendfile()
2892 if (tp->t_tcpreq_closed == 0) { in tcp_log_sendfile()
2898 ent = &tp->t_tcpreq_info[i]; in tcp_log_sendfile()
2899 if (ent->flags == TCP_TRK_TRACK_FLG_EMPTY) { in tcp_log_sendfile()
2903 if (ent->flags & TCP_TRK_TRACK_FLG_OPEN) { in tcp_log_sendfile()
2907 if (ent->flags & TCP_TRK_TRACK_FLG_COMP) { in tcp_log_sendfile()
2912 if ((ent->start == offset) || in tcp_log_sendfile()
2913 ((offset > ent->start) && (offset < ent->end))){ in tcp_log_sendfile()
2921 ent->flags |= TCP_TRK_TRACK_FLG_SEQV; in tcp_log_sendfile()
2926 * we must use sb_ccc since the data may still be in-flight in TLS. in tcp_log_sendfile()
2933 if (SEQ_GT((tp->snd_una + so->so_snd.sb_ccc), ent->end_seq)) in tcp_log_sendfile()
2934 ent->end_seq = tp->snd_una + so->so_snd.sb_ccc; in tcp_log_sendfile()
2935 if ((offset + nbytes) >= ent->end) { in tcp_log_sendfile()
2936 ent->flags |= TCP_TRK_TRACK_FLG_COMP; in tcp_log_sendfile()
2949 ent = &tp->t_tcpreq_info[i]; in tcp_log_sendfile()
2950 if (ent->flags == TCP_TRK_TRACK_FLG_EMPTY) { in tcp_log_sendfile()
2954 if ((ent->flags & TCP_TRK_TRACK_FLG_OPEN) == 0) in tcp_log_sendfile()
2957 if (ent->start == offset) { in tcp_log_sendfile()
2959 ent->start_seq = tp->snd_una + in tcp_log_sendfile()
2960 tptosocket(tp)->so_snd.sb_ccc; in tcp_log_sendfile()
2961 ent->flags |= TCP_TRK_TRACK_FLG_SEQV; in tcp_log_sendfile()
2963 } else if (offset > ent->start) { in tcp_log_sendfile()
2964 ent->flags |= TCP_TRK_TRACK_FLG_SEQV; in tcp_log_sendfile()