Lines Matching defs:bbr

271 static int32_t bbr_sack_not_required = 0;	/* set to one to allow non-sack to use bbr */
432 static inline uint64_t bbr_get_bw(struct tcp_bbr *bbr);
438 bbr_get_pacing_length(struct tcp_bbr *bbr, uint16_t gain,
441 bbr_get_a_state_target(struct tcp_bbr *bbr, uint32_t gain);
443 bbr_set_state(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t win);
445 bbr_set_probebw_gains(struct tcp_bbr *bbr,  uint32_t cts, uint32_t losses);
447 bbr_substate_change(struct tcp_bbr *bbr, uint32_t cts, int line,
450 bbr_get_target_cwnd(struct tcp_bbr *bbr, uint64_t bw, uint32_t gain);
452 bbr_state_change(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch,
455 bbr_calc_thresh_rack(struct tcp_bbr *bbr, uint32_t srtt, uint32_t cts,
458 bbr_initial_cwnd(struct tcp_bbr *bbr, struct tcpcb *tp);
460 bbr_calc_thresh_tlp(struct tcpcb *tp, struct tcp_bbr *bbr,
463 bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts,
466 bbr_set_state_target(struct tcp_bbr *bbr, int line);
468 bbr_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts, int32_t line);
470 bbr_log_progress_event(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t tick,
473 tcp_bbr_tso_size_check(struct tcp_bbr *bbr, uint32_t cts);
475 bbr_setup_red_bw(struct tcp_bbr *bbr, uint32_t cts);
477 bbr_log_rtt_shrinks(struct tcp_bbr *bbr, uint32_t cts, uint32_t applied,
481 bbr_find_lowest_rsm(struct tcp_bbr *bbr);
483 bbr_get_rtt(struct tcp_bbr *bbr, int32_t rtt_type);
485 bbr_log_to_start(struct tcp_bbr *bbr, uint32_t cts, uint32_t to, int32_t slot,
488 bbr_log_timer_var(struct tcp_bbr *bbr, int mode, uint32_t cts,
492 bbr_log_hpts_diag(struct tcp_bbr *bbr, uint32_t cts, struct hpts_diag *diag);
494 bbr_log_type_bbrsnd(struct tcp_bbr *bbr, uint32_t len, uint32_t slot,
498 bbr_enter_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts,
501 bbr_stop_all_timers(struct tcpcb *tp, struct tcp_bbr *bbr);
503 bbr_exit_probe_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts);
505 bbr_check_probe_rtt_limits(struct tcp_bbr *bbr, uint32_t cts);
507 bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts);
509 bbr_log_pacing_delay_calc(struct tcp_bbr *bbr, uint16_t gain, uint32_t len,
515 bbr_state_val(struct tcp_bbr *bbr)
517 	return(bbr->rc_bbr_substate);
521 get_min_cwnd(struct tcp_bbr *bbr)
525 	mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options),
526 		  bbr->r_ctl.rc_pace_max_segs);
527 	if (bbr_get_rtt(bbr, BBR_RTT_PROP) < BBR_HIGH_SPEED)
534 bbr_get_persists_timer_val(struct tcpcb *tp, struct tcp_bbr *bbr)
539 	bbr->r_ctl.rc_hpts_flags |= PACE_TMR_PERSIT;
553 bbr_timer_start(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
566 	if (bbr->rc_all_timers_stopped) {
570 	if (bbr->rc_in_persist) {
572 		return (bbr_get_persists_timer_val(tp, bbr));
574 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
585 			rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
588 				if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time))
591 					tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time;
595 			bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RXT;
606 				tov = bbr->r_ctl.rc_min_to;
608 			    (bbr->r_ctl.rc_min_rto_ms * MS_IN_USEC),
609 			    (bbr->rc_max_rto_sec * USECS_IN_SECOND));
610 			bbr_log_timer_var(bbr, 2, cts, 0, bbr_get_rtt(bbr, BBR_SRTT), 0, to);
616 		rsm = bbr_find_lowest_rsm(bbr);
628 			 * We don't start a bbr rack timer if all we have is
633 		srtt = bbr_get_rtt(bbr, BBR_RTT_RACK);
634 		thresh = bbr_calc_thresh_rack(bbr, srtt, cts, rsm);
639 			if (to < bbr->r_ctl.rc_min_to) {
640 				to = bbr->r_ctl.rc_min_to;
643 			to = bbr->r_ctl.rc_min_to;
647 		if (bbr->rc_tlp_in_progress != 0) {
653 		rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext);
665 		if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time))
668 			tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time;
672 		srtt = bbr_get_rtt(bbr, bbr_tlp_type_to_use);
673 		thresh = bbr_calc_thresh_tlp(tp, bbr, rsm, srtt, cts);
677 			to = bbr->r_ctl.rc_min_to;
678 		if (to > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {
685 		if ((bbr->rc_tlp_rtx_out == 1) &&
686 		    (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq)) {
691 			bbr->rc_tlp_rtx_out = 0;
694 		if (rsm->r_start != bbr->r_ctl.rc_last_tlp_seq) {
699 			bbr->r_ctl.rc_tlp_seg_send_cnt = 0;
700 			bbr->r_ctl.rc_last_tlp_seq = rsm->r_start;
705 		bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RACK;
707 		bbr_log_timer_var(bbr, 1, cts, time_since_sent, srtt, thresh, to);
708 		if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) {
716 			bbr->r_ctl.rc_hpts_flags |= PACE_TMR_TLP;
723 bbr_minseg(struct tcp_bbr *bbr)
725 	return (bbr->r_ctl.rc_pace_min_segs - bbr->rc_last_options);
729 bbr_start_hpts_timer(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts, int32_t frm, int32_t slot, uint32_t tot_len)
748 	stopped = bbr->rc_tmr_stopped;
749 	if (stopped && TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) {
750 		left = bbr->r_ctl.rc_timer_exp - cts;
752 	bbr->r_ctl.rc_hpts_flags = 0;
753 	bbr->r_ctl.rc_timer_exp = 0;
754 	prev_delay = bbr->r_ctl.rc_last_delay_val;
755 	if (bbr->r_ctl.rc_last_delay_val &&
762 		slot = bbr->r_ctl.rc_last_delay_val;
763 		if (TSTMP_GT(cts, bbr->rc_pacer_started)) {
765 			delay_calc = cts - bbr->rc_pacer_started;
771 	if (bbr->r_agg_early_set) {
772 		bbr_log_pacing_delay_calc(bbr, 0, bbr->r_ctl.rc_agg_early, cts, slot, 0, bbr->r_agg_early_set, 2);
773 		slot += bbr->r_ctl.rc_agg_early;
774 		bbr->r_ctl.rc_agg_early = 0;
775 		bbr->r_agg_early_set = 0;
778 	if (bbr->r_ctl.rc_hptsi_agg_delay) {
779 		if (slot > bbr->r_ctl.rc_hptsi_agg_delay) {
781 			slot -= bbr->r_ctl.rc_hptsi_agg_delay;
782 			bbr->r_ctl.rc_hptsi_agg_delay = 0;
785 			bbr->r_ctl.rc_hptsi_agg_delay -= slot;
786 			bbr->r_ctl.rc_last_delay_val = slot = 100;
789 	bbr->r_ctl.rc_last_delay_val = slot;
790 	hpts_timeout = bbr_timer_start(tp, bbr, cts);
792 		if (bbr->rc_in_persist == 0) {
812 		bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;
818 		bbr->r_ctl.rc_hpts_flags |= PACE_PKT_OUTPUT;
838 			bbr->r_ctl.rc_hpts_flags |= PACE_TMR_KEEP;
842 	    (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK)) {
854 	if (bbr->r_ctl.rc_incr_tmrs && slot &&
855 	    (bbr->r_ctl.rc_hpts_flags & (PACE_TMR_TLP|PACE_TMR_RXT))) {
881 		bbr->r_ctl.rc_timer_exp = cts + hpts_timeout;
883 		bbr->r_ctl.rc_timer_exp = 0;
885 	    (bbr->rc_use_google ||
886 	     bbr->output_error_seen ||
892 		bbr->rc_tp->t_flags2 |= TF2_MBUF_QUEUE_READY;
893 		if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) &&
894 		    (bbr->rc_cwnd_limited == 0)) {
903 		bbr->rc_pacer_started = cts;
907 		bbr->rc_timer_first = 0;
908 		bbr->bbr_timer_src = frm;
909 		bbr_log_to_start(bbr, cts, hpts_timeout, slot, 1);
910 		bbr_log_hpts_diag(bbr, cts, &diag);
925 			bbr->rc_pacer_started = cts;
926 		if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) &&
927 		    (bbr->rc_cwnd_limited == 0)) {
940 		bbr->bbr_timer_src = frm;
941 		bbr_log_to_start(bbr, cts, hpts_timeout, slot, 0);
942 		bbr_log_hpts_diag(bbr, cts, &diag);
943 		bbr->rc_timer_first = 1;
945 	bbr->rc_tmr_stopped = 0;
946 	bbr_log_type_bbrsnd(bbr, tot_len, slot, delay_calc, cts, frm, prev_delay);
950 bbr_timer_audit(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, struct sockbuf *sb)
962 	tmr_up = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK;
963 	if (bbr->rc_in_persist && (tmr_up == PACE_TMR_PERSIT))
965 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
971 	inp = bbr->rc_inp;
1042 	if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) {
1045 		bbr_timer_cancel(bbr, __LINE__, cts);
1046 		bbr_start_hpts_timer(bbr, tp, cts, 1, bbr->r_ctl.rc_last_delay_val,
1057 		hpts_timeout = bbr_timer_start(tp, bbr, cts);
1061 				bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;
1065 				bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;
1071 			bbr->r_ctl.rc_timer_exp = cts + hpts_timeout;
1352 	    "Do we allow bbr to run on connections not supporting SACK?");
1849 bbr_fill_in_logging_data(struct tcp_bbr *bbr, struct tcp_log_bbr *l, uint32_t cts)
1852 	l->cur_del_rate = bbr->r_ctl.rc_bbr_cur_del_rate;
1853 	l->delRate = get_filter_value(&bbr->r_ctl.rc_delrate);
1854 	l->rttProp = get_filter_value_small(&bbr->r_ctl.rc_rttprop);
1855 	l->bw_inuse = bbr_get_bw(bbr);
1856 	l->inflight = ctf_flight_size(bbr->rc_tp,
1857 			  (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
1858 	l->applimited = bbr->r_ctl.r_app_limited_until;
1859 	l->delivered = bbr->r_ctl.rc_delivered;
1861 	l->lost = bbr->r_ctl.rc_lost;
1862 	l->bbr_state = bbr->rc_bbr_state;
1863 	l->bbr_substate = bbr_state_val(bbr);
1864 	l->epoch = bbr->r_ctl.rc_rtt_epoch;
1865 	l->lt_epoch = bbr->r_ctl.rc_lt_epoch;
1866 	l->pacing_gain = bbr->r_ctl.rc_bbr_hptsi_gain;
1867 	l->cwnd_gain = bbr->r_ctl.rc_bbr_cwnd_gain;
1868 	l->inhpts = tcp_in_hpts(bbr->rc_tp);
1869 	l->use_lt_bw = bbr->rc_lt_use_bw;
1870 	l->pkts_out = bbr->r_ctl.rc_flight_at_input;
1871 	l->pkt_epoch = bbr->r_ctl.rc_pkt_epoch;
1875 bbr_log_type_bw_reduce(struct tcp_bbr *bbr, int reason)
1877 	if (tcp_bblogging_on(bbr->rc_tp)) {
1880 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
1885 		log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_loss_rate;
1887 		log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_enters_probertt;
1889 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1890 		    &bbr->rc_inp->inp_socket->so_rcv,
1891 		    &bbr->rc_inp->inp_socket->so_snd,
1893 		    0, &log, false, &bbr->rc_tv);
1898 bbr_log_type_rwnd_collapse(struct tcp_bbr *bbr, int seq, int mode, uint32_t count)
1900 	if (tcp_bblogging_on(bbr->rc_tp)) {
1903 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
1907 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1908 		    &bbr->rc_inp->inp_socket->so_rcv,
1909 		    &bbr->rc_inp->inp_socket->so_snd,
1911 		    0, &log, false, &bbr->rc_tv);
1916 bbr_log_type_just_return(struct tcp_bbr *bbr, uint32_t cts, uint32_t tlen, uint8_t hpts_calling,
1919 	if (tcp_bblogging_on(bbr->rc_tp)) {
1922 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
1924 		log.u_bbr.flex2 = bbr->r_ctl.rc_hpts_flags;
1925 		log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp;
1927 		log.u_bbr.flex5 = bbr->rc_in_persist;
1928 		log.u_bbr.flex6 = bbr->r_ctl.rc_last_delay_val;
1930 		log.u_bbr.flex8 = bbr->rc_in_persist;
1933 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1934 		    &bbr->rc_inp->inp_socket->so_rcv,
1935 		    &bbr->rc_inp->inp_socket->so_snd,
1937 		    tlen, &log, false, &bbr->rc_tv);
1942 bbr_log_type_enter_rec(struct tcp_bbr *bbr, uint32_t seq)
1944 	if (tcp_bblogging_on(bbr->rc_tp)) {
1947 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
1949 		log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent;
1950 		log.u_bbr.flex3 = bbr->r_ctl.rc_recovery_start;
1951 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1952 		    &bbr->rc_inp->inp_socket->so_rcv,
1953 		    &bbr->rc_inp->inp_socket->so_snd,
1955 		    0, &log, false, &bbr->rc_tv);
1960 bbr_log_msgsize_fail(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t len, uint32_t maxseg, uint32_t mtu, int32_t csum_flags, int32_t tso, uint32_t cts)
1965 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
1971 		    &bbr->rc_inp->inp_socket->so_rcv,
1972 		    &bbr->rc_inp->inp_socket->so_snd,
1974 		    0, &log, false, &bbr->rc_tv);
1979 bbr_log_flowend(struct tcp_bbr *bbr)
1981 	if (tcp_bblogging_on(bbr->rc_tp)) {
1986 		if (bbr->rc_inp->inp_socket) {
1987 			r = &bbr->rc_inp->inp_socket->so_rcv;
1988 			s = &bbr->rc_inp->inp_socket->so_snd;
1992 		bbr_fill_in_logging_data(bbr, &log.u_bbr, tcp_get_usecs(&tv));
1993 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2001 bbr_log_pkt_epoch(struct tcp_bbr *bbr, uint32_t cts, uint32_t line,
2004 	if (tcp_bblogging_on(bbr->rc_tp)) {
2007 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2010 		log.u_bbr.flex3 = bbr->r_ctl.rc_bbr_lastbtlbw;
2011 		log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_rtt;
2012 		log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch;
2013 		log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;
2016 		log.u_bbr.inflight = bbr->r_ctl.r_measurement_count;
2017 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2018 		    &bbr->rc_inp->inp_socket->so_rcv,
2019 		    &bbr->rc_inp->inp_socket->so_snd,
2021 		    0, &log, false, &bbr->rc_tv);
2026 bbr_log_time_epoch(struct tcp_bbr *bbr, uint32_t cts, uint32_t line, uint32_t epoch_time)
2028 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2031 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2032 		log.u_bbr.flex1 = bbr->r_ctl.rc_lost;
2033 		log.u_bbr.flex2 = bbr->rc_inp->inp_socket->so_snd.sb_lowat;
2034 		log.u_bbr.flex3 = bbr->rc_inp->inp_socket->so_snd.sb_hiwat;
2036 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2037 		    &bbr->rc_inp->inp_socket->so_rcv,
2038 		    &bbr->rc_inp->inp_socket->so_snd,
2040 		    0, &log, false, &bbr->rc_tv);
2045 bbr_log_set_of_state_target(struct tcp_bbr *bbr, uint32_t new_tar, int line, int meth)
2047 	if (tcp_bblogging_on(bbr->rc_tp)) {
2050 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2051 		log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state;
2054 		log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs;
2056 		log.u_bbr.flex6 = bbr->r_ctl.rc_pace_min_segs;
2057 		log.u_bbr.flex7 = bbr->rc_last_options;
2059 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2060 		    &bbr->rc_inp->inp_socket->so_rcv,
2061 		    &bbr->rc_inp->inp_socket->so_snd,
2063 		    0, &log, false, &bbr->rc_tv);
2069 bbr_log_type_statechange(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
2071 	if (tcp_bblogging_on(bbr->rc_tp)) {
2074 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2076 		log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;
2077 		log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int;
2079 			log.u_bbr.flex4 = bbr_get_rtt(bbr, BBR_RTT_PKTRTT);
2081 			log.u_bbr.flex4 = bbr_get_rtt(bbr, BBR_RTT_PROP);
2082 		log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch;
2083 		log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;
2084 		log.u_bbr.flex7 = (bbr->r_ctl.rc_target_at_state/1000);
2085 		log.u_bbr.lt_epoch = bbr->r_ctl.rc_level_state_extra;
2086 		log.u_bbr.pkts_out = bbr->r_ctl.rc_target_at_state;
2087 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2088 		    &bbr->rc_inp->inp_socket->so_rcv,
2089 		    &bbr->rc_inp->inp_socket->so_snd,
2091 		    0, &log, false, &bbr->rc_tv);
2096 bbr_log_rtt_shrinks(struct tcp_bbr *bbr, uint32_t cts, uint32_t applied,
2099 	if (tcp_bblogging_on(bbr->rc_tp)) {
2102 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2104 		log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;
2105 		log.u_bbr.flex3 = bbr->r_ctl.last_in_probertt;
2108 		log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state;
2111 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2112 		    &bbr->rc_inp->inp_socket->so_rcv,
2113 		    &bbr->rc_inp->inp_socket->so_snd,
2115 		    0, &log, false, &bbr->rc_tv);
2120 bbr_log_type_exit_rec(struct tcp_bbr *bbr)
2122 	if (tcp_bblogging_on(bbr->rc_tp)) {
2125 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2126 		log.u_bbr.flex1 = bbr->r_ctl.rc_recovery_start;
2127 		log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent;
2128 		log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2129 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2130 		    &bbr->rc_inp->inp_socket->so_rcv,
2131 		    &bbr->rc_inp->inp_socket->so_snd,
2133 		    0, &log, false, &bbr->rc_tv);
2138 bbr_log_type_cwndupd(struct tcp_bbr *bbr, uint32_t bytes_this_ack, uint32_t chg,
2141 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2144 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2152 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2153 		    &bbr->rc_inp->inp_socket->so_rcv,
2154 		    &bbr->rc_inp->inp_socket->so_snd,
2156 		    0, &log, false, &bbr->rc_tv);
2161 bbr_log_rtt_sample(struct tcp_bbr *bbr, uint32_t rtt, uint32_t tsin)
2167 	if (tcp_bblogging_on(bbr->rc_tp)) {
2170 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2172 		log.u_bbr.flex2 = bbr->r_ctl.rc_bbr_state_time;
2173 		log.u_bbr.flex3 = bbr->r_ctl.rc_ack_hdwr_delay;
2174 		log.u_bbr.flex4 = bbr->rc_tp->ts_offset;
2175 		log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2176 		log.u_bbr.pkts_out = tcp_tv_to_mssectick(&bbr->rc_tv);
2179 		log.u_bbr.flex8 = bbr->rc_ack_was_delayed;
2180 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2181 		    &bbr->rc_inp->inp_socket->so_rcv,
2182 		    &bbr->rc_inp->inp_socket->so_snd,
2184 		    0, &log, false, &bbr->rc_tv);
2189 bbr_log_type_pesist(struct tcp_bbr *bbr, uint32_t cts, uint32_t time_in, int32_t line, uint8_t enter_exit)
2191 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2194 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2198 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2199 		    &bbr->rc_inp->inp_socket->so_rcv,
2200 		    &bbr->rc_inp->inp_socket->so_snd,
2202 		    0, &log, false, &bbr->rc_tv);
2206 bbr_log_ack_clear(struct tcp_bbr *bbr, uint32_t cts)
2208 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2211 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2212 		log.u_bbr.flex1 = bbr->rc_tp->ts_recent_age;
2213 		log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;
2214 		log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int;
2215 		log.u_bbr.flex4 = bbr->r_ctl.rc_went_idle_time;
2216 		log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2217 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2218 		    &bbr->rc_inp->inp_socket->so_rcv,
2219 		    &bbr->rc_inp->inp_socket->so_snd,
2221 		    0, &log, false, &bbr->rc_tv);
2226 bbr_log_ack_event(struct tcp_bbr *bbr, struct tcphdr *th, struct tcpopt *to, uint32_t tlen,
2229 	if (tcp_bblogging_on(bbr->rc_tp)) {
2233 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2235 		log.u_bbr.flex2 = bbr->r_ctl.rc_lost_bytes;
2263 		log.u_bbr.flex4 = bbr->r_ctl.rc_target_at_state;
2264 		log.u_bbr.flex7 = bbr->r_wanted_output;
2265 		log.u_bbr.flex8 = bbr->rc_in_persist;
2266 		TCP_LOG_EVENTP(bbr->rc_tp, th,
2267 		    &bbr->rc_inp->inp_socket->so_rcv,
2268 		    &bbr->rc_inp->inp_socket->so_snd,
2270 		    tlen, &log, true, &bbr->rc_tv);
2275 bbr_log_doseg_done(struct tcp_bbr *bbr, uint32_t cts, int32_t nxt_pkt, int32_t did_out)
2277 	if (tcp_bblogging_on(bbr->rc_tp)) {
2280 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2283 		log.u_bbr.flex3 = bbr->r_ctl.rc_last_delay_val;
2284 		log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags;
2285 		log.u_bbr.flex5 = bbr->r_ctl.rc_timer_exp;
2286 		log.u_bbr.flex6 = bbr->r_ctl.rc_lost_bytes;
2287 		log.u_bbr.flex7 = bbr->r_wanted_output;
2288 		log.u_bbr.flex8 = bbr->rc_in_persist;
2289 		log.u_bbr.pkts_out = bbr->r_ctl.highest_hdwr_delay;
2290 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2291 		    &bbr->rc_inp->inp_socket->so_rcv,
2292 		    &bbr->rc_inp->inp_socket->so_snd,
2294 		    0, &log, true, &bbr->rc_tv);
2299 bbr_log_enobuf_jmp(struct tcp_bbr *bbr, uint32_t len, uint32_t cts,
2302 	if (tcp_bblogging_on(bbr->rc_tp)) {
2305 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2310 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2311 		    &bbr->rc_inp->inp_socket->so_rcv,
2312 		    &bbr->rc_inp->inp_socket->so_snd,
2314 		    len, &log, true, &bbr->rc_tv);
2319 bbr_log_to_processing(struct tcp_bbr *bbr, uint32_t cts, int32_t ret, int32_t timers, uint8_t hpts_calling)
2321 	if (tcp_bblogging_on(bbr->rc_tp)) {
2324 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2327 		log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp;
2328 		log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags;
2330 		log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state;
2332 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2333 		    &bbr->rc_inp->inp_socket->so_rcv,
2334 		    &bbr->rc_inp->inp_socket->so_snd,
2336 		    0, &log, false, &bbr->rc_tv);
2341 bbr_log_to_event(struct tcp_bbr *bbr, uint32_t cts, int32_t to_num)
2343 	if (tcp_bblogging_on(bbr->rc_tp)) {
2347 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2348 		log.u_bbr.flex1 = bbr->bbr_timer_src;
2350 		log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;
2351 		ar = (uintptr_t)(bbr->r_ctl.rc_resend);
2355 		ar = (uintptr_t)bbr->r_ctl.rc_resend;
2358 		log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2360 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2361 		    &bbr->rc_inp->inp_socket->so_rcv,
2362 		    &bbr->rc_inp->inp_socket->so_snd,
2364 		    0, &log, false, &bbr->rc_tv);
2369 bbr_log_startup_event(struct tcp_bbr *bbr, uint32_t cts, uint32_t flex1, uint32_t flex2, uint32_t flex3, uint8_t reason)
2371 	if (tcp_bblogging_on(bbr->rc_tp)) {
2374 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2379 		log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2380 		log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;
2382 		log.u_bbr.cur_del_rate = bbr->r_ctl.rc_bbr_lastbtlbw;
2383 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2384 		    &bbr->rc_inp->inp_socket->so_rcv,
2385 		    &bbr->rc_inp->inp_socket->so_snd,
2387 		    0, &log, false, &bbr->rc_tv);
2392 bbr_log_hpts_diag(struct tcp_bbr *bbr, uint32_t cts, struct hpts_diag *diag)
2394 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2397 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2419 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2420 		    &bbr->rc_inp->inp_socket->so_rcv,
2421 		    &bbr->rc_inp->inp_socket->so_snd,
2423 		    0, &log, false, &bbr->rc_tv);
2428 bbr_log_timer_var(struct tcp_bbr *bbr, int mode, uint32_t cts, uint32_t time_since_sent, uint32_t srtt,
2431 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2434 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2435 		log.u_bbr.flex1 = bbr->rc_tp->t_rttvar;
2440 		log.u_bbr.flex6 = bbr->rc_tp->t_srtt;
2442 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2443 		    &bbr->rc_inp->inp_socket->so_rcv,
2444 		    &bbr->rc_inp->inp_socket->so_snd,
2446 		    0, &log, false, &bbr->rc_tv);
2451 bbr_log_pacing_delay_calc(struct tcp_bbr *bbr, uint16_t gain, uint32_t len,
2454 	if (tcp_bblogging_on(bbr->rc_tp)) {
2457 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2469 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2470 		    &bbr->rc_inp->inp_socket->so_rcv,
2471 		    &bbr->rc_inp->inp_socket->so_snd,
2473 		    len, &log, false, &bbr->rc_tv);
2478 bbr_log_to_start(struct tcp_bbr *bbr, uint32_t cts, uint32_t to, int32_t slot, uint8_t which)
2480 	if (tcp_bblogging_on(bbr->rc_tp)) {
2483 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2485 		log.u_bbr.flex1 = bbr->bbr_timer_src;
2487 		log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;
2489 		log.u_bbr.flex5 = bbr->rc_tp->t_hpts_slot;
2490 		log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2491 		log.u_bbr.pkts_out = bbr->rc_tp->t_flags2;
2493 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2494 		    &bbr->rc_inp->inp_socket->so_rcv,
2495 		    &bbr->rc_inp->inp_socket->so_snd,
2497 		    0, &log, false, &bbr->rc_tv);
2502 bbr_log_thresh_choice(struct tcp_bbr *bbr, uint32_t cts, uint32_t thresh, uint32_t lro, uint32_t srtt, struct bbr_sendmap *rsm, uint8_t frm)
2504 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2507 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2510 		log.u_bbr.flex3 = bbr->r_ctl.rc_reorder_ts;
2512 		log.u_bbr.flex5 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2514 		log.u_bbr.flex7 = bbr->r_ctl.rc_reorder_shift;
2516 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2517 		    &bbr->rc_inp->inp_socket->so_rcv,
2518 		    &bbr->rc_inp->inp_socket->so_snd,
2520 		    0, &log, false, &bbr->rc_tv);
2525 bbr_log_to_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts, uint8_t hpts_removed)
2527 	if (tcp_bblogging_on(bbr->rc_tp)) {
2530 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2532 		log.u_bbr.flex2 = bbr->bbr_timer_src;
2533 		log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;
2534 		log.u_bbr.flex4 = bbr->rc_in_persist;
2535 		log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2536 		log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2538 		log.u_bbr.pkts_out = bbr->rc_pacer_started;
2539 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2540 		    &bbr->rc_inp->inp_socket->so_rcv,
2541 		    &bbr->rc_inp->inp_socket->so_snd,
2543 		    0, &log, false, &bbr->rc_tv);
2548 bbr_log_tstmp_validation(struct tcp_bbr *bbr, uint64_t peer_delta, uint64_t delta)
2550 	if (tcp_bblogging_on(bbr->rc_tp)) {
2553 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2554 		log.u_bbr.flex1 = bbr->r_ctl.bbr_peer_tsratio;
2559 		log.u_bbr.flex7 = bbr->rc_ts_clock_set;
2560 		log.u_bbr.flex8 = bbr->rc_ts_cant_be_used;
2561 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2562 		    &bbr->rc_inp->inp_socket->so_rcv,
2563 		    &bbr->rc_inp->inp_socket->so_snd,
2565 		    0, &log, false, &bbr->rc_tv);
2570 bbr_log_type_tsosize(struct tcp_bbr *bbr, uint32_t cts, uint32_t tsosz, uint32_t tls, uint32_t old_val, uint32_t maxseg, int hdwr)
2572 	if (tcp_bblogging_on(bbr->rc_tp)) {
2575 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2579 		log.u_bbr.flex4 = bbr->r_ctl.bbr_hptsi_bytes_min;
2582 		log.u_bbr.flex7 = bbr->rc_no_pacing;
2584 		log.u_bbr.flex7 |= bbr->rc_past_init_win;
2586 			log.u_bbr.flex8 = 0x80 | bbr->rc_use_google;
2588 			log.u_bbr.flex8 = bbr->rc_use_google;
2589 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2590 		    &bbr->rc_inp->inp_socket->so_rcv,
2591 		    &bbr->rc_inp->inp_socket->so_snd,
2593 		    0, &log, false, &bbr->rc_tv);
2598 bbr_log_type_rsmclear(struct tcp_bbr *bbr, uint32_t cts, struct bbr_sendmap *rsm,
2601 	if (tcp_bblogging_on(bbr->rc_tp)) {
2604 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2615 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2616 		    &bbr->rc_inp->inp_socket->so_rcv,
2617 		    &bbr->rc_inp->inp_socket->so_snd,
2619 		    0, &log, false, &bbr->rc_tv);
2624 bbr_log_type_bbrupd(struct tcp_bbr *bbr, uint8_t flex8, uint32_t cts,
2630 	if (tcp_bblogging_on(bbr->rc_tp)) {
2633 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2644 		if (bbr->rc_ack_was_delayed)
2645 			log.u_bbr.epoch = bbr->r_ctl.rc_ack_hdwr_delay;
2648 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2649 		    &bbr->rc_inp->inp_socket->so_rcv,
2650 		    &bbr->rc_inp->inp_socket->so_snd,
2652 		    flex2, &log, false, &bbr->rc_tv);
2657 bbr_log_type_ltbw(struct tcp_bbr *bbr, uint32_t cts, int32_t reason,
2661 	if (/*bbr_verbose_logging && */tcp_bblogging_on(bbr->rc_tp)) {
2664 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2669 		log.u_bbr.flex5 = bbr->r_ctl.rc_lt_lost;
2670 		log.u_bbr.flex6 = bbr->r_ctl.rc_lt_del;
2671 		log.u_bbr.flex7 = bbr->rc_lt_is_sampling;
2673 		log.u_bbr.bw_inuse = bbr->r_ctl.rc_lt_bw;
2674 		if (bbr->rc_lt_use_bw == 0)
2675 			log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch;
2677 			log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use;
2678 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2679 		    &bbr->rc_inp->inp_socket->so_rcv,
2680 		    &bbr->rc_inp->inp_socket->so_snd,
2682 		    0, &log, false, &bbr->rc_tv);
2687 bbr_log_progress_event(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t tick, int event, int line)
2689 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2692 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2698 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2699 		    &bbr->rc_inp->inp_socket->so_rcv,
2700 		    &bbr->rc_inp->inp_socket->so_snd,
2702 		    0, &log, false, &bbr->rc_tv);
2707 bbr_type_log_hdwr_pacing(struct tcp_bbr *bbr, const struct ifnet *ifp,
2711 	if (tcp_bblogging_on(bbr->rc_tp)) {
2715 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2723 		log.u_bbr.flex8 = bbr->skip_gain;
2725 		log.u_bbr.flex8 |= bbr->gain_is_limited;
2727 		log.u_bbr.flex8 |= bbr->bbr_hdrw_pacing;
2728 		log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg;
2729 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2730 		    &bbr->rc_inp->inp_socket->so_rcv,
2731 		    &bbr->rc_inp->inp_socket->so_snd,
2733 		    0, &log, false, &bbr->rc_tv);
2738 bbr_log_type_bbrsnd(struct tcp_bbr *bbr, uint32_t len, uint32_t slot, uint32_t del_by, uint32_t cts, uint32_t line, uint32_t prev_delay)
2740 	if (tcp_bblogging_on(bbr->rc_tp)) {
2743 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2748 		log.u_bbr.flex5 = bbr->r_ctl.rc_last_delay_val;
2749 		log.u_bbr.flex6 = bbr->r_ctl.rc_hptsi_agg_delay;
2750 		log.u_bbr.flex7 = (0x0000ffff & bbr->r_ctl.rc_hpts_flags);
2751 		log.u_bbr.flex8 = bbr->rc_in_persist;
2752 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2753 		    &bbr->rc_inp->inp_socket->so_rcv,
2754 		    &bbr->rc_inp->inp_socket->so_snd,
2756 		    len, &log, false, &bbr->rc_tv);
2761 bbr_log_type_bbrrttprop(struct tcp_bbr *bbr, uint32_t t, uint32_t end, uint32_t tsconv, uint32_t cts, int32_t match, uint32_t seq, uint8_t flags)
2763 	if (tcp_bblogging_on(bbr->rc_tp)) {
2766 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2767 		log.u_bbr.flex1 = bbr->r_ctl.rc_delivered;
2769 		log.u_bbr.flex3 = bbr->r_ctl.rc_lowest_rtt;
2775 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2776 		    &bbr->rc_inp->inp_socket->so_rcv,
2777 		    &bbr->rc_inp->inp_socket->so_snd,
2779 		    0, &log, false, &bbr->rc_tv);
2784 bbr_log_exit_gain(struct tcp_bbr *bbr, uint32_t cts, int32_t entry_method)
2786 	if (tcp_bblogging_on(bbr->rc_tp)) {
2789 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2790 		log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state;
2791 		log.u_bbr.flex2 = (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
2792 		log.u_bbr.flex3 = bbr->r_ctl.gain_epoch;
2793 		log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs;
2794 		log.u_bbr.flex5 = bbr->r_ctl.rc_pace_min_segs;
2795 		log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_state_atflight;
2798 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2799 		    &bbr->rc_inp->inp_socket->so_rcv,
2800 		    &bbr->rc_inp->inp_socket->so_snd,
2802 		    0, &log, false, &bbr->rc_tv);
2807 bbr_log_settings_change(struct tcp_bbr *bbr, int settings_desired)
2809 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2812 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2821 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2822 		    &bbr->rc_inp->inp_socket->so_rcv,
2823 		    &bbr->rc_inp->inp_socket->so_snd,
2825 		    0, &log, false, &bbr->rc_tv);
2833 bbr_get_full_bw(struct tcp_bbr *bbr)
2837 	bw = get_filter_value(&bbr->r_ctl.rc_delrate);
2843 bbr_set_pktepoch(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
2848 	if (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_pktepoch)
2849 		lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lost_at_pktepoch;
2852 	del = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_pkt_epoch_del;
2863 	bbr->r_ctl.rc_pkt_epoch_loss_rate =  (uint32_t)calclr;
2864 	if (IN_RECOVERY(bbr->rc_tp->t_flags))
2865 		bbr->r_ctl.recovery_lr += (uint32_t)calclr;
2866 	bbr->r_ctl.rc_pkt_epoch++;
2867 	if (bbr->rc_no_pacing &&
2868 	    (bbr->r_ctl.rc_pkt_epoch >= bbr->no_pacing_until)) {
2869 		bbr->rc_no_pacing = 0;
2870 		tcp_bbr_tso_size_check(bbr, cts);
2872 	bbr->r_ctl.rc_pkt_epoch_rtt = bbr_calc_time(cts, bbr->r_ctl.rc_pkt_epoch_time);
2873 	bbr->r_ctl.rc_pkt_epoch_time = cts;
2875 	bbr_log_pkt_epoch(bbr, cts, line, lost, del);
2876 	bbr->r_ctl.rc_pkt_epoch_del = bbr->r_ctl.rc_delivered;
2877 	bbr->r_ctl.rc_lost_at_pktepoch = bbr->r_ctl.rc_lost;
2881 bbr_set_epoch(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
2886 	bbr->r_ctl.rc_rtt_epoch++;
2887 	epoch_time = cts - bbr->r_ctl.rc_rcv_epoch_start;
2888 	bbr_log_time_epoch(bbr, cts, line, epoch_time);
2889 	bbr->r_ctl.rc_rcv_epoch_start = cts;
2893 bbr_isit_a_pkt_epoch(struct tcp_bbr *bbr, uint32_t cts, struct bbr_sendmap *rsm, int32_t line, int32_t cum_acked)
2895 	if (SEQ_GEQ(rsm->r_delivered, bbr->r_ctl.rc_pkt_epoch_del)) {
2896 		bbr->rc_is_pkt_epoch_now = 1;
2906 __bbr_get_bw(struct tcp_bbr *bbr)
2920 	if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
2922 		rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop);
2925 			min_bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) *
2928 			if (min_bw < bbr->r_ctl.rc_initial_hptsi_bw) {
2929 				min_bw = bbr->r_ctl.rc_initial_hptsi_bw;
2932 		} else if (bbr->rc_tp->t_srtt != 0) {
2934 			rtt = bbr_get_rtt(bbr, BBR_SRTT);
2937 			min_bw = bbr->r_ctl.rc_initial_hptsi_bw;
2942 	if ((bbr->rc_past_init_win == 0) &&
2943 	    (bbr->r_ctl.rc_delivered > bbr_initial_cwnd(bbr, bbr->rc_tp)))
2944 		bbr->rc_past_init_win = 1;
2945 	if ((bbr->rc_use_google)  && (bbr->r_ctl.r_measurement_count >= 1))
2948 	    ((bbr->r_ctl.r_measurement_count < bbr_min_measurements_req) ||
2949 	     (bbr->rc_past_init_win == 0))) {
2953 		rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop);
2960 			bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) *
2963 			if (bw < bbr->r_ctl.rc_initial_hptsi_bw) {
2964 				bw = bbr->r_ctl.rc_initial_hptsi_bw;
2968 			bw = bbr->r_ctl.rc_initial_hptsi_bw;
2978 	if (bbr->rc_lt_use_bw)
2979 		bw = bbr->r_ctl.rc_lt_bw;
2980 	else if (bbr->r_recovery_bw && (bbr->rc_use_google == 0))
2981 		bw = bbr->r_ctl.red_bw;
2983 		bw = get_filter_value(&bbr->r_ctl.rc_delrate);
2994 bbr_get_bw(struct tcp_bbr *bbr)
2998 	bw = __bbr_get_bw(bbr);
3003 bbr_reset_lt_bw_interval(struct tcp_bbr *bbr, uint32_t cts)
3005 	bbr->r_ctl.rc_lt_epoch = bbr->r_ctl.rc_pkt_epoch;
3006 	bbr->r_ctl.rc_lt_time = bbr->r_ctl.rc_del_time;
3007 	bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered;
3008 	bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
3012 bbr_reset_lt_bw_sampling(struct tcp_bbr *bbr, uint32_t cts)
3014 	bbr->rc_lt_is_sampling = 0;
3015 	bbr->rc_lt_use_bw = 0;
3016 	bbr->r_ctl.rc_lt_bw = 0;
3017 	bbr_reset_lt_bw_interval(bbr, cts);
3021 bbr_lt_bw_samp_done(struct tcp_bbr *bbr, uint64_t bw, uint32_t cts, uint32_t timin)
3026 	if (bbr->r_ctl.rc_lt_bw) {
3028 		if (bbr->r_ctl.rc_lt_bw > bw)
3029 			diff = bbr->r_ctl.rc_lt_bw - bw;
3031 			diff = bw - bbr->r_ctl.rc_lt_bw;
3033 		    (diff <= (bbr->r_ctl.rc_lt_bw / bbr_lt_bw_ratio))) {
3037 			saved_bw = (uint32_t)bbr->r_ctl.rc_lt_bw;
3038 			bbr->r_ctl.rc_lt_bw = (bw + bbr->r_ctl.rc_lt_bw) / 2;	/* average of two */
3039 			bbr->rc_lt_use_bw = 1;
3040 			bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
3045 			bbr->r_ctl.rc_lt_epoch_use = bbr->r_ctl.rc_pkt_epoch;
3050 			bbr_log_type_ltbw(bbr, cts, 4, (uint32_t)bw, saved_bw, (uint32_t)diff, timin);
3054 	bbr->r_ctl.rc_lt_bw = bw;
3055 	bbr_reset_lt_bw_interval(bbr, cts);
3056 	bbr_log_type_ltbw(bbr, cts, 5, 0, (uint32_t)bw, 0, timin);
3060 bbr_randomize_extra_state_time(struct tcp_bbr *bbr)
3066 		deduct = bbr->r_ctl.rc_level_state_extra / ran;
3067 		bbr->r_ctl.rc_level_state_extra -= deduct;
3075 bbr_pick_probebw_substate(struct tcp_bbr *bbr, uint32_t cts)
3081 	bbr->r_ctl.rc_exta_time_gd = 0;
3082 	bbr->rc_hit_state_1 = 0;
3083 	bbr->r_ctl.rc_level_state_extra = 0;
3095 	if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP))
3096 		bbr_set_epoch(bbr, cts, __LINE__);
3098 	bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
3103 bbr_lt_bw_sampling(struct tcp_bbr *bbr, uint32_t cts, int32_t loss_detected)
3108 	if (bbr->r_use_policer == 0)
3110 	if (bbr->rc_lt_use_bw) {
3112 		diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use;
3116 			bbr_reset_lt_bw_sampling(bbr, cts);
3117 			if (bbr->rc_filled_pipe) {
3118 				bbr_set_epoch(bbr, cts, __LINE__);
3119 				bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
3120 				bbr_substate_change(bbr, cts, __LINE__, 0);
3121 				bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
3122 				bbr_log_type_statechange(bbr, cts, __LINE__);
3129 				bbr->rc_bbr_state = BBR_STATE_STARTUP;
3130 				bbr_set_epoch(bbr, cts, __LINE__);
3131 				bbr->r_ctl.rc_bbr_state_time = cts;
3132 				bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
3133 				bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;
3134 				bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;
3135 				bbr_set_state_target(bbr, __LINE__);
3136 				bbr_log_type_statechange(bbr, cts, __LINE__);
3139 			bbr_log_type_ltbw(bbr, cts, 0, 0, 0, 0, 0);
3149 			bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered;
3150 			bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
3153 			lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost;
3154 			delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del;
3158 				bbr_log_type_ltbw(bbr, cts, 7, lost, delivered, 0, 0);
3161 				bbr_log_type_ltbw(bbr, cts, 8, lost, delivered, 0, 0);
3173 	if (bbr->rc_lt_is_sampling == 0) {
3177 		bbr_reset_lt_bw_interval(bbr, cts);
3178 		bbr->rc_lt_is_sampling = 1;
3179 		bbr_log_type_ltbw(bbr, cts, 1, 0, 0, 0, 0);
3183 	if (TSTMP_GEQ(bbr->r_ctl.rc_del_time, bbr->r_ctl.rc_lt_time))
3184 		d_time = bbr->r_ctl.rc_del_time - bbr->r_ctl.rc_lt_time;
3189 	if (bbr->r_ctl.r_app_limited_until) {
3191 		bbr_reset_lt_bw_sampling(bbr, cts);
3193 		bbr_log_type_ltbw(bbr, cts, 2, 0, 0, 0, d_time);
3196 	diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch;
3204 		bbr_log_type_ltbw(bbr, cts, 6, 0, 0, 0, d_time);
3215 		bbr_reset_lt_bw_sampling(bbr, cts);
3217 		bbr_log_type_ltbw(bbr, cts, 3, 0, 0, 0, d_time);
3227 		bbr_log_type_ltbw(bbr, cts, 6, 0, 0, 0, d_time);
3231 	lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost;
3232 	delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del;
3235 		bbr_log_type_ltbw(bbr, cts, 6, lost, delivered, 0, d_time);
3241 		bbr_log_type_ltbw(bbr, cts, 6, 0, 0, 0, d_time);
3246 		bbr_reset_lt_bw_sampling(bbr, cts);
3248 		bbr_log_type_ltbw(bbr, cts, 3, 0, 0, 0, d_time);
3255 	bbr_lt_bw_samp_done(bbr, bw, cts, d_time);
3262 bbr_alloc(struct tcp_bbr *bbr)
3269 		bbr->r_ctl.rc_num_maps_alloced++;
3272 	if (bbr->r_ctl.rc_free_cnt) {
3274 		rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);
3275 		TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next);
3276 		bbr->r_ctl.rc_free_cnt--;
3284 bbr_alloc_full_limit(struct tcp_bbr *bbr)
3287 	    (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {
3289 		if (!bbr->alloc_limit_reported) {
3290 			bbr->alloc_limit_reported = 1;
3295 	return (bbr_alloc(bbr));
3300 bbr_alloc_limit(struct tcp_bbr *bbr, uint8_t limit_type)
3307 		    bbr->r_ctl.rc_num_split_allocs >= V_tcp_map_split_limit) {
3309 			if (!bbr->alloc_limit_reported) {
3310 				bbr->alloc_limit_reported = 1;
3318 	rsm = bbr_alloc(bbr);
3321 		bbr->r_ctl.rc_num_split_allocs++;
3327 bbr_free(struct tcp_bbr *bbr, struct bbr_sendmap *rsm)
3331 		bbr->r_ctl.rc_num_split_allocs--;
3334 		bbr->r_ctl.rc_num_small_maps_alloced--;
3335 	if (bbr->r_ctl.rc_tlp_send == rsm)
3336 		bbr->r_ctl.rc_tlp_send = NULL;
3337 	if (bbr->r_ctl.rc_resend == rsm) {
3338 		bbr->r_ctl.rc_resend = NULL;
3340 	if (bbr->r_ctl.rc_next == rsm)
3341 		bbr->r_ctl.rc_next = NULL;
3342 	if (bbr->r_ctl.rc_sacklast == rsm)
3343 		bbr->r_ctl.rc_sacklast = NULL;
3344 	if (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) {
3346 		TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next);
3348 		bbr->r_ctl.rc_free_cnt++;
3351 	bbr->r_ctl.rc_num_maps_alloced--;
3380 bbr_initial_cwnd(struct tcp_bbr *bbr, struct tcpcb *tp)
3384 	if (bbr->rc_init_win) {
3385 		i_cwnd = bbr->rc_init_win * tp->t_maxseg;
3409 bbr_get_raw_target_cwnd(struct tcp_bbr *bbr, uint32_t gain, uint64_t bw)
3414 	if ((get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) ||
3415 	    (bbr_get_full_bw(bbr) == 0)) {
3417 		return (bbr_initial_cwnd(bbr, bbr->rc_tp));
3423 	rtt = bbr_get_rtt(bbr, bbr_cwndtarget_rtt_touse);
3433 bbr_get_target_cwnd(struct tcp_bbr *bbr, uint64_t bw, uint32_t gain)
3437 	mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs);
3439 	cwnd = roundup(bbr_get_raw_target_cwnd(bbr, bw, gain), mss);
3445 	cwnd += (bbr_quanta * bbr->r_ctl.rc_pace_max_segs);
3446 	if (bbr->rc_use_google) {
3447 		if((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&
3448 		   (bbr_state_val(bbr) == BBR_SUB_GAIN)) {
3463 	if (cwnd < get_min_cwnd(bbr))
3464 		return (get_min_cwnd(bbr));
3469 bbr_gain_adjust(struct tcp_bbr *bbr, uint16_t gain)
3477 bbr_get_header_oh(struct tcp_bbr *bbr)
3482 	if (bbr->r_ctl.rc_inc_tcp_oh) {
3484 		seg_oh = (bbr->rc_last_options + sizeof(struct tcphdr));
3486 	if (bbr->r_ctl.rc_inc_ip_oh) {
3489 		if (bbr->r_is_v6) {
3500 	if (bbr->r_ctl.rc_inc_enet_oh) {
3508 bbr_get_pacing_length(struct tcp_bbr *bbr, uint16_t gain, uint32_t useconds_time, uint64_t bw)
3514 	gain = bbr_gain_adjust(bbr, gain);
3529 bbr_get_pacing_delay(struct tcp_bbr *bbr, uint16_t gain, int32_t len, uint32_t cts, int nolog)
3538 	maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
3540 	if (bbr->rc_use_google == 0) {
3541 		seg_oh = bbr_get_header_oh(bbr);
3544 	gain = bbr_gain_adjust(bbr, gain);
3545 	bw = bbr_get_bw(bbr);
3546 	if (bbr->rc_use_google) {
3553 		cbw = bw *  (uint64_t)(1000 - bbr->r_ctl.bbr_google_discount);
3566 	srtt = bbr_get_rtt(bbr, BBR_SRTT);
3568 	    (bbr->rc_use_google == 0) &&
3578 		bbr_log_pacing_delay_calc(bbr, gain, len, cts, usecs, bw, over, 1);
3583 bbr_ack_received(struct tcpcb *tp, struct tcp_bbr *bbr, struct tcphdr *th, uint32_t bytes_this_ack,
3605 		time_stamp = max(1, ((bbr->r_ctl.rc_rcvtime - tp->gput_ts) / 1000));
3618 	if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&
3619 	    ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) {
3623 	maxseg = tp->t_maxseg - bbr->rc_last_options;
3632 		if ((bytes_this_ack < maxseg) && bbr->rc_use_google)
3639 	bw = get_filter_value(&bbr->r_ctl.rc_delrate);
3641 		target_cwnd = bbr_get_target_cwnd(bbr,
3643 						  (uint32_t)bbr->r_ctl.rc_bbr_cwnd_gain);
3645 		target_cwnd = bbr_initial_cwnd(bbr, bbr->rc_tp);
3647 	    (bbr->bbr_prev_in_rec == 0)) {
3652 		bbr->pkt_conservation = 1;
3653 		bbr->r_ctl.rc_recovery_start = bbr->r_ctl.rc_rcvtime;
3655 				       (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
3661 		bbr->bbr_prev_in_rec = 1;
3669 					 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
3670 		bbr_log_type_cwndupd(bbr, flight, 0,
3672 		if (bbr->pkt_conservation) {
3675 			if (TSTMP_GEQ(bbr->r_ctl.rc_rcvtime, bbr->r_ctl.rc_recovery_start))
3676 				time_in = bbr->r_ctl.rc_rcvtime - bbr->r_ctl.rc_recovery_start;
3680 			if (time_in >= bbr_get_rtt(bbr, BBR_RTT_PROP)) {
3682 				bbr->pkt_conservation = 0;
3686 				if (cwnd < get_min_cwnd(bbr))
3687 					cwnd = get_min_cwnd(bbr);
3689 				bbr_log_type_cwndupd(bbr, saved_bytes, sack_changed,
3695 		bbr->bbr_prev_in_rec = 0;
3696 	if ((bbr->rc_use_google == 0) && bbr->r_ctl.restrict_growth) {
3697 		bbr->r_ctl.restrict_growth--;
3701 	if (bbr->rc_filled_pipe) {
3713 		else if (bbr_cwnd_may_shrink || bbr->rc_use_google || bbr->rc_no_pacing)
3721 		    (bbr->rc_past_init_win == 0)) {
3732 	tp->snd_cwnd = max(cwnd, get_min_cwnd(bbr));
3733 	bbr_log_type_cwndupd(bbr, saved_bytes, sack_changed, prev_acked, meth, target_cwnd, th->th_ack, line);
3739 	struct tcp_bbr *bbr;
3741 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
3744 		(bbr->r_ctl.rc_sacked  + bbr->r_ctl.rc_lost_bytes)) <=
3746 		bbr->r_wanted_output = 1;
3753 	struct tcp_bbr *bbr;
3757 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
3763 	bbr->r_recovery_bw = 0;
3765 	tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);
3766 	bbr->pkt_conservation = 0;
3767 	if (bbr->rc_use_google == 0) {
3775 		bbr->bbr_prev_in_rec = 0;
3777 	bbr_log_type_exit_rec(bbr);
3778 	if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {
3779 		tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent);
3780 		bbr_log_type_cwndupd(bbr, 0, 0, 0, 15, 0, 0, __LINE__);
3783 		if (bbr->r_ctl.rc_saved_cwnd < bbr->r_ctl.rc_cwnd_on_ent) {
3784 			bbr->r_ctl.rc_saved_cwnd = bbr->r_ctl.rc_cwnd_on_ent;
3785 			bbr_log_type_cwndupd(bbr, 0, 0, 0, 16, 0, 0, __LINE__);
3789 		     (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
3790 	if ((bbr->rc_use_google == 0) &&
3796 		if (bbr_get_rtt(bbr, BBR_SRTT)) {
3797 			val = ((uint64_t)bbr_get_rtt(bbr, BBR_RTT_PROP) * (uint64_t)1000);
3798 			val /= bbr_get_rtt(bbr, BBR_SRTT);
3803 		bbr_log_type_cwndupd(bbr, bbr_red_mul, bbr_red_div,
3804 				     bbr->r_ctl.recovery_lr, 21,
3806 				     bbr->r_ctl.rc_red_cwnd_pe,
3810 		if (((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&
3813 		     (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&
3814 		     bbr->rc_hit_state_1 &&
3815 		     (bbr_state_val(bbr) == BBR_SUB_DRAIN)) ||
3816 		    ((bbr->rc_bbr_state == BBR_STATE_DRAIN) &&
3822 			cwnd = bbr->r_ctl.rc_saved_cwnd;
3823 			cwnd_p = &bbr->r_ctl.rc_saved_cwnd;
3828 		maxseg = tp->t_maxseg - bbr->rc_last_options;
3830 		if (bbr->r_ctl.rc_lost == 0)
3832 		else if (bbr->r_ctl.rc_delivered == 0)
3835 			lr2use = (uint64_t)bbr->r_ctl.rc_lost * (uint64_t)1000;
3836 			lr2use /= bbr->r_ctl.rc_delivered;
3838 		lr2use += bbr->r_ctl.recovery_lr;
3841 			lr2use *= bbr_get_rtt(bbr, BBR_SRTT);
3844 			    ((bbr_get_rtt(bbr, BBR_SRTT)/bbr_red_scale) > 1))
3845 			    bbr->r_ctl.restrict_growth += acks_inflight;
3865 			if ((newcwnd + (acks_inflight * maxseg)) < get_min_cwnd(bbr)) {
3867 				newcwnd = (get_min_cwnd(bbr) - acks_inflight);
3881 		bbr_log_type_cwndupd(bbr, bbr_red_mul, bbr_red_div, val, 22,
3883 				     bbr_get_rtt(bbr, BBR_SRTT), __LINE__);
3884 		bbr->r_ctl.rc_red_cwnd_pe = bbr->r_ctl.rc_pkt_epoch;
3887 	bbr->r_ctl.recovery_lr = 0;
3889 		bbr->r_wanted_output = 1;
3891 	tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);
3895 bbr_setup_red_bw(struct tcp_bbr *bbr, uint32_t cts)
3897 	bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate);
3899 	if (bbr->r_ctl.red_bw > bbr->r_ctl.rc_bbr_cur_del_rate)
3900 		bbr->r_ctl.red_bw = bbr->r_ctl.rc_bbr_cur_del_rate;
3901 	if (bbr->r_ctl.red_bw < (get_filter_value(&bbr->r_ctl.rc_delrate) / 2))
3902 		bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate) / 2;
3903 	tcp_bbr_tso_size_check(bbr, cts);
3909 	struct tcp_bbr *bbr;
3915 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
3921 			bbr_set_pktepoch(bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
3922 			if (bbr->rc_lt_is_sampling || bbr->rc_lt_use_bw) {
3928 				bbr->r_ctl.rc_lt_epoch++;
3930 			if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
3937 				bbr->r_ctl.rc_bbr_last_startup_epoch++;
3939 			bbr->r_ctl.rc_cwnd_on_ent = tp->snd_cwnd;
3941 			bbr->rc_tlp_rtx_out = 0;
3942 			bbr->r_ctl.recovery_lr = bbr->r_ctl.rc_pkt_epoch_loss_rate;
3943 			tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);
3944 			if (tcp_in_hpts(bbr->rc_tp) &&
3945 			    ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) == 0)) {
3952 				bbr->rc_timer_first = 1;
3953 				bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
3961 			if ((bbr->rc_use_google == 0) &&
3962 			    (bbr->r_ctl.bbr_rttprobe_gain_val ||
3963 			     (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT))) {
3965 					           (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
3966 					(tp->t_maxseg - bbr->rc_last_options);
3967 				if (tp->snd_cwnd < get_min_cwnd(bbr)) {
3969 					tp->snd_cwnd = get_min_cwnd(bbr);
3971 				bbr_log_type_cwndupd(bbr, 0, 0, 0, 14, 0, 0, __LINE__);
3973 			bbr_log_type_enter_rec(bbr, rsm->r_start);
3979 		bbr_reset_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime);
3980 		if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {
3984 			tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent);
3985 			bbr_log_type_cwndupd(bbr, 0, 0, 0, 13, 0, 0, __LINE__);
4006 #define DELAY_ACK(tp, bbr, nsegs)				\
4009 	 ((bbr->bbr_segs_rcvd + nsegs) < tp->t_delayed_ack) &&	\
4020 bbr_find_lowest_rsm(struct tcp_bbr *bbr)
4029 	TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_tmap, r_tnext) {
4040 bbr_find_high_nonack(struct tcp_bbr *bbr, struct bbr_sendmap *rsm)
4050 	TAILQ_FOREACH_REVERSE_FROM(prsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
4065 bbr_calc_thresh_rack(struct tcp_bbr *bbr, uint32_t srtt, uint32_t cts, struct bbr_sendmap *rsm)
4088 	if (bbr->r_ctl.rc_reorder_ts) {
4089 		if (bbr->r_ctl.rc_reorder_fade) {
4090 			if (SEQ_GEQ(cts, bbr->r_ctl.rc_reorder_ts)) {
4091 				lro = cts - bbr->r_ctl.rc_reorder_ts;
4103 			if (lro > bbr->r_ctl.rc_reorder_fade) {
4105 				bbr->r_ctl.rc_reorder_ts = 0;
4115 	thresh = srtt + bbr->r_ctl.rc_pkt_delay;
4118 		if (bbr->r_ctl.rc_reorder_shift)
4119 			thresh += (srtt >> bbr->r_ctl.rc_reorder_shift);
4126 	if ((bbr->rc_tp)->t_srtt == 0)
4129 		t_rxtcur = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
4134 	if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {
4135 		thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND);
4137 	bbr_log_thresh_choice(bbr, cts, thresh, lro, srtt, rsm, BBR_TO_FRM_RACK);
4147 bbr_calc_thresh_tlp(struct tcpcb *tp, struct tcp_bbr *bbr,
4156 	if (bbr->rc_tlp_threshold)
4157 		thresh = srtt + (srtt / bbr->rc_tlp_threshold);
4160 	maxseg = tp->t_maxseg - bbr->rc_last_options;
4197 	bbr_log_thresh_choice(bbr, cts, thresh, t_rxtcur, srtt, rsm, BBR_TO_FRM_TLP);
4203 	if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {
4204 		thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND);
4217 bbr_get_rtt(struct tcp_bbr *bbr, int32_t rtt_type)
4222 	f_rtt = get_filter_value_small(&bbr->r_ctl.rc_rttprop);
4223 	if (get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) {
4225 		if (bbr->rc_tp->t_srtt == 0)
4228 			f_rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);
4241 		if (bbr->r_ctl.rc_pkt_epoch_rtt) {
4242 			srtt = bbr->r_ctl.rc_pkt_epoch_rtt;
4248 		srtt = bbr->r_ctl.rc_last_rtt;
4250 		if (bbr->rc_ack_was_delayed)
4251 			srtt += bbr->r_ctl.rc_ack_hdwr_delay;
4253 		srtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);
4265 bbr_is_lost(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t cts)
4269 	thresh = bbr_calc_thresh_rack(bbr, bbr_get_rtt(bbr, BBR_RTT_RACK),
4282 bbr_check_recovery_mode(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4293 	if (TAILQ_EMPTY(&bbr->r_ctl.rc_map)) {
4297 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
4311 		rsm = bbr_find_lowest_rsm(bbr);
4321 	if (bbr_is_lost(bbr, rsm, cts) &&
4326 			bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;
4327 			bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;
4332 			panic("tp:%p bbr:%p rsm:%p length is 0?", tp, bbr, rsm);
4347 bbr_timeout_rack(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4358 	if (bbr->rc_all_timers_stopped) {
4361 	if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {
4366 	lost = bbr->r_ctl.rc_lost;
4367 	if (bbr->r_state && (bbr->r_state != tp->t_state))
4368 		bbr_set_state(tp, bbr, 0);
4369 	bbr_log_to_event(bbr, cts, BBR_TO_FRM_RACK);
4370 	if (bbr->r_ctl.rc_resend == NULL) {
4372 		bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);
4375 		bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost));
4376 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RACK;
4381 bbr_clone_rsm(struct tcp_bbr *bbr, struct bbr_sendmap *nrsm, struct bbr_sendmap *rsm, uint32_t start)
4411 	idx = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs);
4416 		bbr->r_ctl.rc_num_small_maps_alloced++;
4421 		bbr->r_ctl.rc_num_small_maps_alloced++;
4479 bbr_merge_rsm(struct tcp_bbr *bbr,
4500 		TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, r_rsm, r_tnext);
4513 		bbr->r_ctl.rc_lost_bytes -= r_rsm->r_end - r_rsm->r_start;
4515 	TAILQ_REMOVE(&bbr->r_ctl.rc_map, r_rsm, r_next);
4521 	bbr_free(bbr, r_rsm);
4534 bbr_timeout_tlp(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4546 	if (bbr->rc_all_timers_stopped) {
4549 	if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {
4554 		bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
4558 	if (bbr->rc_in_persist) {
4561 	if (bbr->r_state && (bbr->r_state != tp->t_state))
4562 		bbr_set_state(tp, bbr, 0);
4564 	maxseg = tp->t_maxseg - bbr->rc_last_options;
4588 			bbr->rc_tlp_new_data = 1;
4592 		bbr->r_ctl.rc_tlp_seg_send_cnt = 0;
4593 		bbr->r_ctl.rc_last_tlp_seq = tp->snd_max;
4594 		bbr->r_ctl.rc_tlp_send = NULL;
4605 		rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);
4607 			rsm = bbr_find_high_nonack(bbr, rsm);
4617 		TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
4625 			rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
4636 		nrsm = bbr_alloc_full_limit(bbr);
4645 		bbr_clone_rsm(bbr, nrsm, rsm, (rsm->r_end - maxseg));
4646 		TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
4648 			TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
4655 	bbr->r_ctl.rc_tlp_send = rsm;
4656 	bbr->rc_tlp_rtx_out = 1;
4657 	if (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) {
4658 		bbr->r_ctl.rc_tlp_seg_send_cnt++;
4661 		bbr->r_ctl.rc_last_tlp_seq = rsm->r_start;
4662 		bbr->r_ctl.rc_tlp_seg_send_cnt = 1;
4665 	if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) {
4671 		bbr->rc_tlp_new_data = 0;
4672 		bbr->r_ctl.rc_tlp_send = NULL;
4680 	if (rsm && (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) &&
4681 	    (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend)) {
4688 	bbr_log_to_event(bbr, cts, BBR_TO_FRM_TLP);
4689 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_TLP;
4702 bbr_timeout_delack(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4704 	if (bbr->rc_all_timers_stopped) {
4707 	bbr_log_to_event(bbr, cts, BBR_TO_FRM_DELACK);
4711 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_DELACK;
4723 bbr_timeout_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4728 	if (bbr->rc_all_timers_stopped) {
4731 	if (bbr->rc_in_persist == 0)
4738 	bbr_log_to_event(bbr, cts, BBR_TO_FRM_PERSIST);
4739 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_PERSIT;
4745 		bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
4761 	if ((sbavail(&bbr->rc_inp->inp_socket->so_snd) == 0) &&
4763 		bbr_exit_persist(tp, bbr, cts, __LINE__);
4777 	t_template = tcpip_maketemplate(bbr->rc_inp);
4789 	bbr_start_hpts_timer(bbr, tp, cts, 3, 0, 0);
4801 bbr_timeout_keepalive(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4806 	if (bbr->rc_all_timers_stopped) {
4809 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_KEEP;
4810 	bbr_log_to_event(bbr, cts, BBR_TO_FRM_KEEP);
4841 	bbr_start_hpts_timer(bbr, tp, cts, 4, 0, 0);
4861 	struct tcp_bbr *bbr;
4864 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
4865 	cts = tcp_get_usecs(&bbr->rc_tv);
4866 	lost = bbr->r_ctl.rc_lost;
4867 	if (bbr->r_state && (bbr->r_state != tp->t_state))
4868 		bbr_set_state(tp, bbr, 0);
4870 	TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
4878 					TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
4880 					TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, trsm, rsm, r_tnext);
4887 			bbr_log_type_rsmclear(bbr, cts, rsm, old_flags, __LINE__);
4899 				bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;
4900 				bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;
4924 	bbr->r_ctl.rc_resend = TAILQ_FIRST(&bbr->r_ctl.rc_map);
4926 	bbr_log_to_event(bbr, cts, BBR_TO_FRM_TMR);
4927 	bbr->rc_tlp_new_data = 0;
4928 	bbr->r_ctl.rc_tlp_seg_send_cnt = 0;
4930 	bbr->r_ctl.rc_hptsi_agg_delay = 0;
4931 	bbr->r_agg_early_set = 0;
4932 	bbr->r_ctl.rc_agg_early = 0;
4933 	bbr->rc_tlp_rtx_out = 0;
4934 	bbr->r_ctl.rc_sacked = 0;
4935 	bbr->r_ctl.rc_sacklast = NULL;
4936 	bbr->r_timer_override = 1;
4937 	bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost));
4945 bbr_timeout_rxt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4952 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RXT;
4953 	if (bbr->rc_all_timers_stopped) {
4967 		bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
4971 	if ((bbr->r_ctl.rc_resend == NULL) ||
4972 	    ((bbr->r_ctl.rc_resend->r_flags & BBR_RWND_COLLAPSED) == 0)) {
5006 		tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;
5016 		tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;
5018 		tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;
5028 	    MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms),
5029 	    MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000));
5077 			isipv6 = bbr->r_is_v6;
5159 		if (bbr->r_is_v6)
5167 	sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
5176 bbr_process_timers(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, uint8_t hpts_calling)
5179 	int32_t timers = (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK);
5186 		if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)
5190 	if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {
5193 		if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
5195 			bbr_log_to_processing(bbr, cts, ret, 0, hpts_calling);
5200 			bbr_log_to_processing(bbr, cts, ret, 0, hpts_calling);
5207 		left = bbr->r_ctl.rc_timer_exp - cts;
5209 		bbr_log_to_processing(bbr, cts, ret, left, hpts_calling);
5213 	bbr->rc_tmr_stopped = 0;
5214 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_MASK;
5216 		ret = bbr_timeout_delack(tp, bbr, cts);
5218 		ret = bbr_timeout_persist(tp, bbr, cts);
5220 		bbr->r_ctl.rc_tlp_rxt_last_time = cts;
5221 		ret = bbr_timeout_rack(tp, bbr, cts);
5223 		bbr->r_ctl.rc_tlp_rxt_last_time = cts;
5224 		ret = bbr_timeout_tlp(tp, bbr, cts);
5226 		bbr->r_ctl.rc_tlp_rxt_last_time = cts;
5227 		ret = bbr_timeout_rxt(tp, bbr, cts);
5229 		ret = bbr_timeout_keepalive(tp, bbr, cts);
5231 	bbr_log_to_processing(bbr, cts, ret, timers, hpts_calling);
5236 bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts)
5238 	if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
5241 		if (tcp_in_hpts(bbr->rc_tp) &&
5242 		    (bbr->rc_timer_first == 1)) {
5249 			tcp_hpts_remove(bbr->rc_tp);
5250 			if (bbr->r_ctl.rc_last_delay_val) {
5254 				if (TSTMP_GT(cts, bbr->rc_pacer_started))
5255 					time_since_send = cts - bbr->rc_pacer_started;
5258 				if (bbr->r_ctl.rc_last_delay_val > time_since_send) {
5260 					bbr->r_ctl.rc_last_delay_val -= time_since_send;
5262 					bbr->r_ctl.rc_last_delay_val = 0;
5264 				bbr->rc_pacer_started = cts;
5267 		bbr->rc_timer_first = 0;
5268 		bbr_log_to_cancel(bbr, line, cts, hpts_removed);
5269 		bbr->rc_tmr_stopped = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK;
5270 		bbr->r_ctl.rc_hpts_flags &= ~(PACE_TMR_MASK);
5277 	struct tcp_bbr *bbr;
5279 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
5280 	bbr->rc_all_timers_stopped = 1;
5288 bbr_get_earliest_send_outstanding(struct tcp_bbr *bbr, struct bbr_sendmap *u_rsm, uint32_t cts)
5292 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
5299 bbr_update_rsm(struct tcpcb *tp, struct tcp_bbr *bbr,
5317 		bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
5331 		bbr->r_ctl.rc_holes_rxt += (rsm->r_end - rsm->r_start);
5337 	rsm->r_delivered = bbr->r_ctl.rc_delivered;
5338 	rsm->r_ts_valid = bbr->rc_ts_valid;
5339 	if (bbr->rc_ts_valid)
5340 		rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts;
5341 	if (bbr->r_ctl.r_app_limited_until)
5345 	if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)
5346 		rsm->r_bbr_state = bbr_state_val(bbr);
5355 		bbr_log_type_rsmclear(bbr, cts, rsm, old_flags, __LINE__);
5356 		bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
5357 		if (bbr->r_ctl.rc_sacked == 0)
5358 			bbr->r_ctl.rc_sacklast = NULL;
5361 		TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
5363 	TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
5370 	rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts);
5371 	rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp,
5372 						(bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
5373 	bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next);
5374 	if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) {
5377 	} else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) {
5384 	rsm->r_del_time = bbr->r_ctl.rc_del_time; /* TEMP GOOGLE CODE */
5394 bbr_update_entry(struct tcpcb *tp, struct tcp_bbr *bbr,
5412 		bbr_update_rsm(tp, bbr, rsm, cts, pacing_time);
5430 	nrsm = bbr_alloc_full_limit(bbr);
5442 	bbr_clone_rsm(bbr, nrsm, rsm, c_end);
5443 	TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
5446 		TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
5450 	bbr_update_rsm(tp, bbr, rsm, cts, pacing_time);
5456 bbr_get_hardware_rate(struct tcp_bbr *bbr)
5460 	bw = bbr_get_bw(bbr);
5467 bbr_setup_less_of_rate(struct tcp_bbr *bbr, uint32_t cts,
5474 	if (get_filter_value(&bbr->r_ctl.rc_delrate) >= act_rate) {
5478 		bbr->skip_gain = 1;
5479 		bbr->gain_is_limited = 0;
5480 		red = get_filter_value(&bbr->r_ctl.rc_delrate) - act_rate;
5482 			filter_reduce_by(&bbr->r_ctl.rc_delrate, red, cts);
5485 		bbr->skip_gain = 0;
5486 		bbr->gain_is_limited = 1;
5491 bbr_update_hardware_pacing_rate(struct tcp_bbr *bbr, uint32_t cts)
5496 	if (bbr->r_ctl.crte == NULL)
5498 	if ((bbr->rc_inp->inp_route.ro_nh == NULL) ||
5499 	    (bbr->rc_inp->inp_route.ro_nh->nh_ifp == NULL)) {
5502 		bbr->bbr_attempt_hdwr_pace = 0;
5504 		bbr->gain_is_limited = 0;
5505 		bbr->skip_gain = 0;
5506 		bbr->bbr_hdrw_pacing = 0;
5509 		tcp_bbr_tso_size_check(bbr, cts);
5512 	rate = bbr_get_hardware_rate(bbr);
5513 	nrte = tcp_chg_pacing_rate(bbr->r_ctl.crte,
5514 				   bbr->rc_tp,
5515 				   bbr->rc_inp->inp_route.ro_nh->nh_ifp,
5522 	if (nrte != bbr->r_ctl.crte) {
5523 		bbr->r_ctl.crte = nrte;
5526 			if (bbr->r_ctl.crte->rate < rate) {
5528 				bbr_setup_less_of_rate(bbr, cts,
5529 						       bbr->r_ctl.crte->rate, rate);
5532 				bbr->gain_is_limited = 0;
5533 				bbr->skip_gain = 0;
5538 			bbr->gain_is_limited = 0;
5539 			bbr->skip_gain = 0;
5540 			bbr->bbr_hdrw_pacing = 0;
5542 		bbr_type_log_hdwr_pacing(bbr,
5543 					 bbr->r_ctl.crte->ptbl->rs_ifp,
5545 					 bbr->r_ctl.crte->rate,
5553 bbr_adjust_for_hw_pacing(struct tcp_bbr *bbr, uint32_t cts)
5563 	if ((bbr->bbr_hdrw_pacing == 0) ||
5564 	    (IN_RECOVERY(bbr->rc_tp->t_flags)) ||
5565 	    (bbr->r_ctl.crte == NULL))
5567 	if (bbr->hw_pacing_set == 0) {
5575 	rlp = bbr->r_ctl.crte;
5576 	if (bbr->rc_tp->t_maxseg > bbr->rc_last_options)
5577 		maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
5579 		maxseg = BBR_MIN_SEG - bbr->rc_last_options;
5586 	cur_delay = bbr_get_pacing_delay(bbr, BBR_UNIT,
5587 		        bbr->r_ctl.rc_pace_max_segs, cts, 1);
5588 	hdwr_delay = bbr->r_ctl.rc_pace_max_segs / maxseg;
5594 	bbr_log_type_tsosize(bbr, cts, delta, cur_delay, hdwr_delay,
5595 			     (bbr->r_ctl.rc_pace_max_segs / maxseg),
5599 			  bbr->r_ctl.bbr_hptsi_segments_delay_tar)))) {
5614 			     (bbr->r_ctl.rc_pace_max_segs/maxseg));
5617 		    (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) {
5623 			seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg;
5637 		seg_sz = bbr->r_ctl.rc_pace_max_segs * bbr_hdwr_pace_adjust;
5639 		    (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) {
5645 			seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg;
5659 		seg_sz = bbr->r_ctl.rc_pace_max_segs;
5661 	if (seg_sz > bbr->r_ctl.rc_pace_max_segs)
5664 		new_tso = bbr->r_ctl.rc_pace_max_segs;
5668 	if (new_tso != bbr->r_ctl.rc_pace_max_segs) {
5669 		bbr_log_type_tsosize(bbr, cts, new_tso, 0, bbr->r_ctl.rc_pace_max_segs, maxseg, 0);
5670 		bbr->r_ctl.rc_pace_max_segs = new_tso;
5675 tcp_bbr_tso_size_check(struct tcp_bbr *bbr, uint32_t cts)
5734 	 * This will get you "google bbr" behavior with respect to tso size.
5741 	if (bbr->rc_tp->t_maxseg > bbr->rc_last_options) {
5742 		maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
5744 		maxseg = BBR_MIN_SEG - bbr->rc_last_options;
5746 	old_tso = bbr->r_ctl.rc_pace_max_segs;
5747 	if (bbr->rc_past_init_win == 0) {
5753 		if (bbr->rc_use_google)
5754 			bbr->r_ctl.rc_pace_max_segs = ((bbr->rc_tp->t_maxseg - bbr->rc_last_options) * 2);
5755 		else if (bbr->bbr_init_win_cheat)
5756 			bbr->r_ctl.rc_pace_max_segs = bbr_initial_cwnd(bbr, bbr->rc_tp);
5758 			bbr->r_ctl.rc_pace_max_segs = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
5759 		if (bbr->r_ctl.rc_pace_min_segs != bbr->rc_tp->t_maxseg)
5760 			bbr->r_ctl.rc_pace_min_segs = bbr->rc_tp->t_maxseg;
5761 		if (bbr->r_ctl.rc_pace_max_segs == 0) {
5762 			bbr->r_ctl.rc_pace_max_segs = maxseg;
5764 		bbr_log_type_tsosize(bbr, cts, bbr->r_ctl.rc_pace_max_segs, tls_seg, old_tso, maxseg, 0);
5765 			bbr_adjust_for_hw_pacing(bbr, cts);
5773 	bw = bbr_get_bw(bbr);
5774 	if (IN_RECOVERY(bbr->rc_tp->t_flags) &&
5775 	     (bbr->rc_use_google == 0)) {
5778 	} else if (bbr->rc_use_google) {
5782 		if (bbr->r_ctl.rc_bbr_hptsi_gain != BBR_UNIT) {
5783 			bw *= bbr->r_ctl.rc_bbr_hptsi_gain;
5797 	} else if (bbr->rc_no_pacing) {
5799 	} else if (bw <= bbr->r_ctl.bbr_cross_over) {
5806 		tso_len = bbr_get_pacing_length(bbr, BBR_UNIT, bbr->r_ctl.bbr_hptsi_segments_delay_tar, bw);
5809 			if (new_tso > bbr->r_ctl.bbr_hptsi_segments_max)
5810 				new_tso = bbr->r_ctl.bbr_hptsi_segments_max;
5817 			min_tso = bbr_minseg(bbr);
5843 		bw /= bbr->r_ctl.bbr_hptsi_per_second;
5850 		if (new_tso < (bbr->r_ctl.bbr_hptsi_segments_max * maxseg))
5851 			new_tso = (bbr->r_ctl.bbr_hptsi_segments_max * maxseg);
5853 	if (bbr->r_ctl.bbr_hptsi_segments_floor && (new_tso < (maxseg * bbr->r_ctl.bbr_hptsi_segments_floor)))
5854 		new_tso = maxseg * bbr->r_ctl.bbr_hptsi_segments_floor;
5858 	if (bbr->r_ctl.bbr_utter_max && (new_tso > (bbr->r_ctl.bbr_utter_max * maxseg))) {
5859 		new_tso = bbr->r_ctl.bbr_utter_max * maxseg;
5863 		bbr_log_type_tsosize(bbr, cts, new_tso, tls_seg, old_tso, maxseg, 0);
5864 		bbr->r_ctl.rc_pace_max_segs = new_tso;
5867 	bbr_adjust_for_hw_pacing(bbr, cts);
5871 bbr_log_output(struct tcp_bbr *bbr, struct tcpcb *tp, struct tcpopt *to, int32_t len,
5942 	pacing_time = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, len, cts, 1);
5946 		rsm = bbr_alloc(bbr);
5961 		rsm->r_delivered = bbr->r_ctl.rc_delivered;
5963 		rsm->r_ts_valid = bbr->rc_ts_valid;
5964 		if (bbr->rc_ts_valid)
5965 			rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts;
5966 		rsm->r_del_time = bbr->r_ctl.rc_del_time;
5967 		if (bbr->r_ctl.r_app_limited_until)
5971 		rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts);
5972 		rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp,
5973 						(bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
5979 		TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next);
5980 		TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
5982 		if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)
5983 			rsm->r_bbr_state = bbr_state_val(bbr);
5986 		if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) {
5989 		} else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) {
6005 	} else if (bbr->r_ctl.rc_next) {
6007 		rsm = bbr->r_ctl.rc_next;
6017 		seq_out = bbr_update_entry(tp, bbr, rsm, cts, &len, pacing_time);
6025 	TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
6027 			seq_out = bbr_update_entry(tp, bbr, rsm, cts, &len, pacing_time);
6028 			bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next);
6041 			nrsm = bbr_alloc_full_limit(bbr);
6043 				bbr_update_rsm(tp, bbr, rsm, cts, pacing_time);
6050 			bbr_clone_rsm(bbr, nrsm, rsm, seq_out);
6051 			TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
6053 				TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
6057 			seq_out = bbr_update_entry(tp, bbr, nrsm, cts, &len, pacing_time);
6074 		TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
6080 		    bbr, tp);
6095 bbr_collapse_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, int32_t rtt)
6105 tcp_bbr_xmit_timer(struct tcp_bbr *bbr, uint32_t rtt_usecs, uint32_t rsm_send_time, uint32_t r_start, uint32_t tsin)
6107 	bbr->rtt_valid = 1;
6108 	bbr->r_ctl.cur_rtt = rtt_usecs;
6109 	bbr->r_ctl.ts_in = tsin;
6111 		bbr->r_ctl.cur_rtt_send_time = rsm_send_time;
6115 bbr_make_timestamp_determination(struct tcp_bbr *bbr)
6118 	 * We have in our bbr control:
6119 	 * 1) The timestamp we started observing cum-acks (bbr->r_ctl.bbr_ts_check_tstmp).
6120 	 * 2) Our timestamp indicating when we sent that packet (bbr->r_ctl.rsm->bbr_ts_check_our_cts).
6121 	 * 3) The current timestamp that just came in (bbr->r_ctl.last_inbound_ts)
6122 	 * 4) The time that the packet that generated that ack was sent (bbr->r_ctl.cur_rtt_send_time)
6126 	 * delta = bbr->r_ctl.cur_rtt_send_time - bbr->r_ctl.bbr_ts_check_our_cts
6130 	 * peer_delta = bbr->r_ctl.last_inbound_ts - bbr->r_ctl.bbr_ts_check_tstmp
6141 	 * timestamp difference which we will store in bbr->r_ctl.bbr_peer_tsratio.
6155 	delta = bbr->r_ctl.cur_rtt_send_time - bbr->r_ctl.bbr_ts_check_our_cts;
6165 	peer_delta = bbr->r_ctl.last_inbound_ts - bbr->r_ctl.bbr_ts_check_tstmp;
6188 			bbr->rc_ts_clock_set = 1;
6189 			bbr->r_ctl.bbr_peer_tsratio = 1;
6191 			bbr->rc_ts_cant_be_used = 1;
6192 			bbr->rc_ts_clock_set = 1;
6197 	bbr->rc_ts_clock_set = 1;
6202 		bbr->r_ctl.bbr_peer_tsratio = 1;
6203 		bbr_log_tstmp_validation(bbr, peer_delta, delta);
6209 		bbr->r_ctl.bbr_peer_tsratio = 10;
6210 		bbr_log_tstmp_validation(bbr, peer_delta, delta);
6216 		bbr->r_ctl.bbr_peer_tsratio = 100;
6217 		bbr_log_tstmp_validation(bbr, peer_delta, delta);
6223 		bbr->r_ctl.bbr_peer_tsratio = 1000;
6224 		bbr_log_tstmp_validation(bbr, peer_delta, delta);
6230 		bbr->r_ctl.bbr_peer_tsratio = 10000;
6234 	bbr->rc_ts_cant_be_used = 1;
6235 	bbr->r_ctl.bbr_peer_tsratio = 0;
6236 	bbr_log_tstmp_validation(bbr, peer_delta, delta);
6244 tcp_bbr_xmit_timer_commit(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts)
6250 	if (bbr->rtt_valid == 0)
6254 	rtt = bbr->r_ctl.cur_rtt;
6255 	tsin = bbr->r_ctl.ts_in;
6256 	if (bbr->rc_prtt_set_ts) {
6270 		bbr->rc_prtt_set_ts = 0;
6271 		rtt_prop = get_filter_value_small(&bbr->r_ctl.rc_rttprop);
6273 			filter_increase_by_small(&bbr->r_ctl.rc_rttprop, (rtt - rtt_prop), cts);
6275 			apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
6280 	if (bbr->rc_ack_was_delayed)
6281 		rtt += bbr->r_ctl.rc_ack_hdwr_delay;
6283 	if (rtt < bbr->r_ctl.rc_lowest_rtt)
6284 		bbr->r_ctl.rc_lowest_rtt = rtt;
6285 	bbr_log_rtt_sample(bbr, rtt, tsin);
6286 	if (bbr->r_init_rtt) {
6291 		bbr->r_init_rtt = 0;
6294 	if ((bbr->rc_ts_clock_set == 0) && bbr->rc_ts_valid) {
6302 		if (bbr->rc_ack_is_cumack) {
6303 			if (bbr->rc_ts_data_set) {
6305 				bbr_make_timestamp_determination(bbr);
6307 				bbr->rc_ts_data_set = 1;
6308 				bbr->r_ctl.bbr_ts_check_tstmp = bbr->r_ctl.last_inbound_ts;
6309 				bbr->r_ctl.bbr_ts_check_our_cts = bbr->r_ctl.cur_rtt_send_time;
6316 			bbr->rc_ts_data_set = 0;
6379 	    max(MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms), rtt_ticks + 2),
6380 	    MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000));
6390 	rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);
6391 	if (bbr->r_ctl.bbr_smallest_srtt_this_state > rtt)
6392 		bbr->r_ctl.bbr_smallest_srtt_this_state = rtt;
6396 bbr_set_reduced_rtt(struct tcp_bbr *bbr, uint32_t cts, uint32_t line)
6398 	bbr->r_ctl.rc_rtt_shrinks = cts;
6400 	    (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) &&
6401 	    ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) {
6406 		bbr_enter_probe_rtt(bbr, cts, __LINE__);
6408 		bbr_check_probe_rtt_limits(bbr, cts);
6412 tcp_bbr_commit_bw(struct tcp_bbr *bbr, uint32_t cts)
6416 	if (bbr->r_ctl.rc_bbr_cur_del_rate == 0) {
6418 		bbr_log_type_bbrupd(bbr, 20, cts, 0, 0,
6422 	if (bbr->r_ctl.r_measurement_count < 0xffffffff)
6423 		bbr->r_ctl.r_measurement_count++;
6424 	orig_bw = get_filter_value(&bbr->r_ctl.rc_delrate);
6425 	apply_filter_max(&bbr->r_ctl.rc_delrate, bbr->r_ctl.rc_bbr_cur_del_rate, bbr->r_ctl.rc_pkt_epoch);
6426 	bbr_log_type_bbrupd(bbr, 21, cts, (uint32_t)orig_bw,
6427 			    (uint32_t)get_filter_value(&bbr->r_ctl.rc_delrate),
6430 	    (orig_bw != get_filter_value(&bbr->r_ctl.rc_delrate))) {
6431 		if (bbr->bbr_hdrw_pacing) {
6436 			bbr_update_hardware_pacing_rate(bbr, cts);
6438 		bbr_set_state_target(bbr, __LINE__);
6439 		tcp_bbr_tso_size_check(bbr, cts);
6440 		if (bbr->r_recovery_bw)  {
6441 			bbr_setup_red_bw(bbr, cts);
6442 			bbr_log_type_bw_reduce(bbr, BBR_RED_BW_USELRBW);
6444 	} else if ((orig_bw == 0) && get_filter_value(&bbr->r_ctl.rc_delrate))
6445 		tcp_bbr_tso_size_check(bbr, cts);
6449 bbr_nf_measurement(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts)
6451 	if (bbr->rc_in_persist == 0) {
6457 		if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time))
6458 			tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time);
6466 		delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered);
6479 		    bbr->rc_ts_valid &&
6480 		    bbr->rc_ts_clock_set &&
6481 		    (bbr->rc_ts_cant_be_used == 0) &&
6482 		    bbr->rc_use_ts_limit) {
6483 			ts_diff = max((bbr->r_ctl.last_inbound_ts - rsm->r_del_ack_ts), 1);
6484 			ts_diff *= bbr->r_ctl.bbr_peer_tsratio;
6488 				bbr_log_type_bbrupd(bbr, 61, cts,
6490 						    bbr->r_ctl.last_inbound_ts,
6497 				bbr_log_type_bbrupd(bbr, 62, cts,
6501 				if ((bbr->ts_can_raise) &&
6503 					bbr_log_type_bbrupd(bbr, 8, cts,
6511 					bbr_log_type_bbrupd(bbr, 7, cts,
6541 				bbr_log_type_bbrupd(bbr, 6, cts,
6552 		bbr->r_ctl.rc_bbr_cur_del_rate = bw;
6554 		    (bw > get_filter_value(&bbr->r_ctl.rc_delrate))) {
6555 			tcp_bbr_commit_bw(bbr, cts);
6556 			bbr_log_type_bbrupd(bbr, 10, cts, (uint32_t)tim, delivered,
6557 					    0, 0, 0, 0,  bbr->r_ctl.rc_del_time,  rsm->r_del_time);
6563 bbr_google_measurement(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts)
6565 	if (bbr->rc_in_persist == 0) {
6572 		if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time))
6573 			tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time);
6581 		delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered);
6585 		if (tim < bbr->r_ctl.rc_lowest_rtt) {
6586 			bbr_log_type_bbrupd(bbr, 99, cts, (uint32_t)tim, delivered,
6587 					    tim, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0);
6595 		bbr->r_ctl.rc_bbr_cur_del_rate = bw;
6617 				bbr_log_type_bbrupd(bbr, 6, cts,
6627 			    (sti < bbr->r_ctl.rc_lowest_rtt)) {
6628 				bbr_log_type_bbrupd(bbr, 99, cts, (uint32_t)tim, delivered,
6629 						    (uint32_t)sti, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0);
6634 		bbr->r_ctl.rc_bbr_cur_del_rate = bw;
6637 		     (bw > get_filter_value(&bbr->r_ctl.rc_delrate)))) {
6638 			tcp_bbr_commit_bw(bbr, cts);
6639 			bbr_log_type_bbrupd(bbr, 10, cts, (uint32_t)tim, delivered,
6640 					    0, 0, 0, 0, bbr->r_ctl.rc_del_time,  rsm->r_del_time);
6646 bbr_update_bbr_info(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts, uint32_t tsin,
6652 	bbr->r_ctl.rc_delivered += (rsm->r_end - rsm->r_start);
6653 	bbr->r_ctl.rc_del_time = cts;
6661 	if ((bbr->rc_use_google == 0) &&
6674 	if ((bbr_no_retran && bbr->rc_use_google) &&
6684 	tcp_bbr_xmit_timer(bbr, rtt, rsm_send_time, rsm->r_start, tsin);
6686 		bbr->rc_ack_is_cumack = 1;
6688 		bbr->rc_ack_is_cumack = 0;
6689 	old_rttprop = bbr_get_rtt(bbr, BBR_RTT_PROP);
6701 		bbr_log_rtt_shrinks(bbr, cts, 0, rtt, __LINE__, BBR_RTTS_NEWRTT, 0);
6702 		bbr_set_reduced_rtt(bbr, cts, __LINE__);
6704 	bbr_log_type_bbrrttprop(bbr, rtt, rsm->r_end, uts, cts,
6706 	apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
6707 	if (old_rttprop != bbr_get_rtt(bbr, BBR_RTT_PROP)) {
6712 		bbr_set_state_target(bbr, __LINE__);
6713 		if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT)
6714 			bbr_log_rtt_shrinks(bbr, cts, 0, 0,
6716 		else if (old_rttprop < bbr_get_rtt(bbr, BBR_RTT_PROP))
6718 			bbr_check_probe_rtt_limits(bbr, cts);
6720 	if ((bbr->rc_use_google == 0) &&
6729 	if (bbr->r_ctl.r_app_limited_until &&
6730 	    (bbr->r_ctl.rc_delivered >= bbr->r_ctl.r_app_limited_until)) {
6732 		bbr->r_ctl.r_app_limited_until = 0;
6734 	if (bbr->rc_use_google) {
6735 		bbr_google_measurement(bbr, rsm, rtt, cts);
6737 		bbr_nf_measurement(bbr, rsm, rtt, cts);
6743  * uses (milliseconds) into one that bbr uses
6760 bbr_update_rtt(struct tcpcb *tp, struct tcp_bbr *bbr,
6784 		bbr->r_ctl.rc_last_rtt = t;
6785 		bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0,
6791 	    (bbr->rc_use_google == 1) &&
6795 		t = tcp_tv_to_mssectick(&bbr->rc_tv) - to->to_tsecr;
6799 		bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0,
6824 				bbr->r_ctl.rc_last_rtt = t;
6825 				bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_TSMATCHING,
6831 					bbr->rc_tlp_rtx_out = 0;
6853 	if (t < bbr->r_ctl.rc_lowest_rtt) {
6868 				panic("rsm:%p bbr:%p rsm has overmax and only 1 retranmit flags:%x?", rsm, bbr, rsm->r_flags);
6875 			bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_EARLIER_RET,
6884 			bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts,
6893 			bbr->rc_tlp_rtx_out = 0;
6895 			bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts,
6898 			bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts,
6910     struct tcp_bbr *bbr, struct bbr_sendmap *rsm)
6915 	TAILQ_FOREACH_REVERSE_FROM(nrsm, &bbr->r_ctl.rc_tmap,
6936 		    bbr_is_lost(bbr, nrsm, bbr->r_ctl.rc_rcvtime)) {
6937 			bbr->r_ctl.rc_lost += nrsm->r_end - nrsm->r_start;
6938 			bbr->r_ctl.rc_lost_bytes += nrsm->r_end - nrsm->r_start;
6950 bbr_proc_sack_blk(struct tcpcb *tp, struct tcp_bbr *bbr, struct sackblk *sack,
6968 		TAILQ_FOREACH_REVERSE_FROM(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
6981 	TAILQ_FOREACH_FROM(rsm, &bbr->r_ctl.rc_map, r_next) {
7000 			nrsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);
7012 			panic("tp:%p bbr:%p sack:%p to:%p prsm:%p",
7013 			    tp, bbr, sack, to, prsm);
7029 			nrsm = bbr_alloc_full_limit(bbr);
7031 			nrsm = bbr_alloc_limit(bbr, BBR_LIMIT_TYPE_SPLIT);
7038 			sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk);
7041 		bbr_clone_rsm(bbr, nrsm, rsm, start);
7042 		TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
7044 			TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
7056 			bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_SACKED, 0);
7058 			bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);
7059 			bbr_log_sack_passed(tp, bbr, rsm);
7061 				bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7066 				bbr->r_ctl.rc_reorder_ts = cts;
7068 					bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7069 					if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7071 						bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7077 				TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7081 		bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_SACKED);
7099 		nrsm = bbr_alloc_full_limit(bbr);
7101 		nrsm = bbr_alloc_limit(bbr, BBR_LIMIT_TYPE_SPLIT);
7108 		sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk);
7112 	bbr_clone_rsm(bbr, nrsm, rsm, end);
7115 	TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
7117 		TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
7121 	bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_SACKED, 0);
7122 	bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_SACKED);
7124 	bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);
7125 	bbr_log_sack_passed(tp, bbr, rsm);
7128 		bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7132 		bbr->r_ctl.rc_reorder_ts = cts;
7134 			bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7135 			if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7137 				bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7143 		TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7157 			rsm = bbr_merge_rsm(bbr, rsm, nrsm);
7164 			rsm = bbr_merge_rsm(bbr, nrsm, rsm);
7181 		bbr->r_ctl.rc_sacklast = TAILQ_NEXT(rsm, r_next);
7183 		bbr->r_ctl.rc_sacklast = NULL;
7189 bbr_peer_reneges(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, tcp_seq th_ack)
7198 		bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
7201 			panic("bbr:%p rsm:%p flags:0x%x in tmap?",
7202 			    bbr, rsm, rsm->r_flags);
7207 			bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7208 			bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7209 			if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7211 				bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7216 		bbr_log_type_rsmclear(bbr, bbr->r_ctl.rc_rcvtime, rsm, oflags, __LINE__);
7219 			TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7222 			TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, tmap, rsm, r_tnext);
7240 	sack_filter_clear(&bbr->r_ctl.bbr_sf, th_ack);
7246 	struct tcp_bbr *bbr;
7250 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
7251 	cts = bbr->r_ctl.rc_rcvtime;
7252 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7256 			bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
7258 			TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);
7260 				TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7263 			if (bbr->r_ctl.rc_next == rsm) {
7265 				bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7268 				bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_CUM_ACKED, 0);
7269 			bbr_free(bbr, rsm);
7288 	struct tcp_bbr *bbr;
7301 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
7302 	cts = bbr->r_ctl.rc_rcvtime;
7304 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7306 	maxseg = tp->t_maxseg - bbr->rc_last_options;
7307 	p_maxseg = min(bbr->r_ctl.rc_pace_max_segs, maxseg);
7310 		bbr_log_progress_event(bbr, tp, ticks, PROGRESS_UPDATE, __LINE__);
7311 		bbr->rc_tp->t_acktime = ticks;
7333 			now = bbr_ts_convert(tcp_tv_to_mssectick(&bbr->rc_tv));
7337 			bbr_log_type_bbrrttprop(bbr, rtt,
7340 			apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
7342 			bbr->r_wanted_output = 1;
7355 		bbr->r_wanted_output = 1;
7363 			panic("No rack map tp:%p for th:%p state:%d bbr:%p snd_una:%u snd_max:%u chg:%d\n",
7365 			    th, tp->t_state, bbr,
7374 		printf("Rack map starts at r_start:%u for th_ack:%u huh? ts:%d rs:%d bbr:%p\n",
7377 		    bbr->r_state, bbr);
7378 		panic("th-ack is bad bbr:%p tp:%p", bbr, tp);
7402 			bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
7403 			if (bbr->r_ctl.rc_sacked == 0)
7404 				bbr->r_ctl.rc_sacklast = NULL;
7406 			bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_CUM_ACKED, th_ack);
7408 				bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7417 				bbr->r_ctl.rc_reorder_ts = cts;
7419 					bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7420 					if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7422 						bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7427 		bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
7429 		TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);
7431 			TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7434 		if (bbr->r_ctl.rc_next == rsm) {
7436 			bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7438 		bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_CUM_ACKED);
7442 		bbr_free(bbr, rsm);
7444 			rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7455 		bbr->r_ctl.rc_sacked -= (th_ack - rsm->r_start);
7456 		if (bbr->r_ctl.rc_sacked == 0)
7457 			bbr->r_ctl.rc_sacklast = NULL;
7466 		bbr_update_rtt(tp, bbr, &lrsm, to, cts, BBR_CUM_ACKED, th_ack);
7475 		bbr->r_ctl.rc_lost_bytes -= th_ack - rsm->r_start;
7477 	bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_CUM_ACKED);
7478 	bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
7484 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7498 		bbr_peer_reneges(bbr, rsm, th->th_ack);
7504 	rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);
7525 			if ((bbr->r_ctl.rc_num_small_maps_alloced > bbr_sack_block_limit) &&
7551 	new_sb = sack_filter_blks(tp, &bbr->r_ctl.bbr_sf, sack_blocks,
7553 	ctf_log_sack_filter(bbr->rc_tp, new_sb, sack_blocks);
7608 	rsm = bbr->r_ctl.rc_sacklast;
7610 		acked = bbr_proc_sack_blk(tp, bbr, &sack_blocks[i], to, &rsm, cts);
7612 			bbr->r_wanted_output = 1;
7626 		rsm = bbr_check_recovery_mode(tp, bbr, cts);
7630 			bbr->r_wanted_output = 1;
7635 			if (bbr->r_ctl.rc_resend == NULL) {
7636 				bbr->r_ctl.rc_resend = rsm;
7646 		if (bbr->r_ctl.rc_resend == NULL) {
7647 			bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);
7659 bbr_strike_dupack(struct tcp_bbr *bbr)
7663 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
7667 			bbr->r_wanted_output = 1;
7689 	struct tcp_bbr *bbr;
7692 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
7693 	lost = bbr->r_ctl.rc_lost;
7728 			bbr->r_wanted_output = 1;
7734 		bbr->r_wanted_output = 1;
7739 		if (bbr->rc_in_persist)
7742 			bbr_strike_dupack(bbr);
7745 	bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, (bbr->r_ctl.rc_lost > lost));
7752 			bbr_ack_received(tp, bbr, th, 0, sack_changed, prev_acked, __LINE__, 0);
7753 			if (bbr->r_state == TCPS_SYN_SENT) {
7815 		bbr_collapse_rtt(tp, bbr, TCP_REXMTVAL(tp));
7818 	bbr_ack_received(tp, bbr, th, acked, sack_changed, prev_acked, __LINE__, (bbr->r_ctl.rc_lost - lost));
7836 		bbr_log_progress_event(bbr, tp, ticks, PROGRESS_CLEAR, __LINE__);
7838 			bbr->rc_tp->t_acktime = 0;
7843 		bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
7844 		if (bbr->rc_in_persist == 0) {
7845 			bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime;
7847 		sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
7848 		bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime);
7871 		bbr->r_wanted_output = 1;
7879 bbr_enter_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, int32_t line)
7881 	if (bbr->rc_in_persist == 0) {
7882 		bbr_timer_cancel(bbr, __LINE__, cts);
7883 		bbr->r_ctl.rc_last_delay_val = 0;
7885 		bbr->rc_in_persist = 1;
7886 		bbr->r_ctl.rc_went_idle_time = cts;
7888 		bbr_log_type_pesist(bbr, cts, 0, line, 1);
7890 		if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
7893 			time_in = cts - bbr->r_ctl.rc_bbr_state_time;
7894 			if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {
7897 				idx = bbr_state_val(bbr);
7900 				counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
7903 		bbr->r_ctl.rc_bbr_state_time = cts;
7908 bbr_restart_after_idle(struct tcp_bbr *bbr, uint32_t cts, uint32_t idle_time)
7916 		if (bbr->rc_use_idle_restart) {
7917 			bbr->rc_bbr_state = BBR_STATE_IDLE_EXIT;
7924 			bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
7925 			bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
7926 			bbr_set_state_target(bbr, __LINE__);
7928 			bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;
7929 			bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;
7930 			bbr_log_type_statechange(bbr, cts, __LINE__);
7931 		} else if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {
7932 			bbr_substate_change(bbr, cts, __LINE__, 1);
7938 bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, int32_t line)
7942 	if (bbr->rc_in_persist == 0)
7944 	idle_time = bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time);
7945 	bbr->rc_in_persist = 0;
7946 	bbr->rc_hit_state_1 = 0;
7947 	bbr->r_ctl.rc_del_time = cts;
7953 	if (tcp_in_hpts(bbr->rc_tp)) {
7954 		tcp_hpts_remove(bbr->rc_tp);
7955 		bbr->rc_timer_first = 0;
7956 		bbr->r_ctl.rc_hpts_flags = 0;
7957 		bbr->r_ctl.rc_last_delay_val = 0;
7958 		bbr->r_ctl.rc_hptsi_agg_delay = 0;
7959 		bbr->r_agg_early_set = 0;
7960 		bbr->r_ctl.rc_agg_early = 0;
7962 	bbr_log_type_pesist(bbr, cts, idle_time, line, 0);
7969 		bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_PERSIST, 0);
7970 		bbr->r_ctl.last_in_probertt = bbr->r_ctl.rc_rtt_shrinks = cts;
7978 	bbr_set_epoch(bbr, cts, __LINE__);
7983 	bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
7985 	bbr->r_ctl.rc_bbr_state_time = cts;
7986 	if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) ||
7987 	    (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT)) {
7993 		bbr_restart_after_idle(bbr, cts, idle_time);
7998 bbr_collapsed_window(struct tcp_bbr *bbr)
8022 	maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
8023 	max_seq = bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd;
8024 	bbr_log_type_rwnd_collapse(bbr, max_seq, 1, 0);
8025 	TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
8035 	bbr->rc_has_collapsed = 0;
8059 		} else if (bbr->r_ctl.rc_num_small_maps_alloced < bbr_sack_block_limit) {
8077 		nrsm = bbr_alloc_limit(bbr, BBR_LIMIT_TYPE_SPLIT);
8084 		bbr_log_type_rwnd_collapse(bbr, max_seq, 3, 0);
8085 		bbr_clone_rsm(bbr, nrsm, rsm, max_seq);
8086 		TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
8088 			TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
8102 	TAILQ_FOREACH_FROM(nrsm, &bbr->r_ctl.rc_map, r_next) {
8105 		bbr->rc_has_collapsed = 1;
8107 	bbr_log_type_rwnd_collapse(bbr, max_seq, 4, fnd);
8111 bbr_un_collapse_window(struct tcp_bbr *bbr)
8116 	TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
8124 	bbr_log_type_rwnd_collapse(bbr,
8125 				   (bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd), 0, cleared);
8126 	bbr->rc_has_collapsed = 0;
8145 	struct tcp_bbr *bbr;
8147 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8163 		bbr->r_wanted_output = 1;
8173 		bbr_collapsed_window(bbr);
8174  	else if (bbr->rc_has_collapsed)
8175 		bbr_un_collapse_window(bbr);
8177 	if ((bbr->rc_in_persist != 0) &&
8178 	    (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2),
8179 				bbr_minseg(bbr)))) {
8184 		bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8187 		bbr->r_wanted_output = 1;
8190 	if ((bbr->rc_in_persist == 0) &&
8191 	    (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
8197 		bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8258 			if (DELAY_ACK(tp, bbr, nsegs) || tfo_syn) {
8259 				bbr->bbr_segs_rcvd += max(1, nsegs);
8261 				bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8263 				bbr->r_wanted_output = 1;
8367 				bbr_timer_cancel(bbr,
8368 				    __LINE__, bbr->r_ctl.rc_rcvtime);
8400 			bbr->rc_timer_first = 1;
8401 			bbr_timer_cancel(bbr,
8402 			    __LINE__, bbr->r_ctl.rc_rcvtime);
8412 		bbr->r_wanted_output = 1;
8430 	struct tcp_bbr *bbr;
8436 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8443 	if (bbr->r_ctl.rc_resend != NULL) {
8464 		tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
8537 	if (DELAY_ACK(tp, bbr, nsegs)) {
8538 		bbr->bbr_segs_rcvd += max(1, nsegs);
8540 		bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8542 		bbr->r_wanted_output = 1;
8567 	struct tcp_bbr *bbr;
8593 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8594 	if (__predict_false(bbr->r_ctl.rc_resend != NULL)) {
8598 	if (__predict_false(bbr->rc_in_persist != 0)) {
8602 	if (bbr->r_ctl.rc_sacked) {
8614 	bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, 0);
8623 	if ((bbr->rc_in_persist != 0) &&
8624 	    (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2),
8625 			       bbr_minseg(bbr)))) {
8626 		bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8627 		bbr->r_wanted_output = 1;
8630 	if ((bbr->rc_in_persist == 0) &&
8631 	    (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
8637 		bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8646 		tp->ts_recent_age = bbr->r_ctl.rc_rcvtime;
8682 		bbr_collapse_rtt(tp, bbr, TCP_REXMTVAL(tp));
8686 		bbr_collapsed_window(bbr);
8687 	else if (bbr->rc_has_collapsed)
8688 		bbr_un_collapse_window(bbr);
8693 	bbr_ack_received(tp, bbr, th, acked, sack_changed, prev_acked, __LINE__, 0);
8710 		bbr_log_progress_event(bbr, tp, ticks, PROGRESS_CLEAR, __LINE__);
8712 			bbr->rc_tp->t_acktime = 0;
8713 		bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8714 		if (bbr->rc_in_persist == 0) {
8715 			bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime;
8717 		sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
8718 		bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime);
8724 		bbr->r_wanted_output = 1;
8727 		bbr->r_wanted_output = 1;
8744 	struct tcp_bbr *bbr;
8749 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8814 		if (DELAY_ACK(tp, bbr, 1) && tlen != 0 && !tfo_partial) {
8815 			bbr->bbr_segs_rcvd += 1;
8817 			bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8819 			bbr->r_wanted_output = 1;
8896 			t = tcp_tv_to_mssectick(&bbr->rc_tv);
8903 				tcp_bbr_xmit_timer(bbr, rtt, 0, 0, 0);
8904 				apply_filter_min_small(&bbr->r_ctl.rc_rttprop,
8905 						       rtt, bbr->r_ctl.rc_rcvtime);
8955 	struct tcp_bbr *bbr;
8959 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8984 			if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RXT) ||
8985 			    (bbr->r_ctl.rc_hpts_flags & PACE_TMR_TLP) ||
8986 			    (bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK)) {
9037 		tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9070 		t = tcp_tv_to_mssectick(&bbr->rc_tv);
9077 			tcp_bbr_xmit_timer(bbr, rtt, 0, 0, 0);
9078 			apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, bbr->r_ctl.rc_rcvtime);
9174 	struct tcp_bbr *bbr;
9193 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9194 	if (bbr->r_ctl.rc_delivered < (4 * tp->t_maxseg)) {
9200 		bbr->r_ctl.rc_init_rwnd = max(tiwin, tp->snd_wnd);
9261 		tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9275 			bbr->r_wanted_output = 1;
9290 			bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
9310 	struct tcp_bbr *bbr;
9315 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9358 		tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9372 			bbr->r_wanted_output = 1;
9387 			bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
9397 bbr_check_data_after_close(struct mbuf *m, struct tcp_bbr *bbr,
9401 	if (bbr->rc_allow_data_af_clo == 0) {
9416 	bbr->r_wanted_output = 1;
9433 	struct tcp_bbr *bbr;
9437 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9469 	    bbr_check_data_after_close(m, bbr, tp, &tlen, th, so))
9489 		tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9503 			bbr->r_wanted_output = 1;
9537 			bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
9558 	struct tcp_bbr *bbr;
9562 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9605 		tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9619 			bbr->r_wanted_output = 1;
9639 			bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
9660 	struct tcp_bbr *bbr;
9664 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9707 		tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9721 			bbr->r_wanted_output = 1;
9741 			bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
9762 	struct tcp_bbr *bbr;
9766 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9801 	    bbr_check_data_after_close(m, bbr, tp, &tlen, th, so))
9821 		tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9835 			bbr->r_wanted_output = 1;
9850 			bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
9860 bbr_stop_all_timers(struct tcpcb *tp, struct tcp_bbr *bbr)
9867 		bbr->rc_in_persist = 1;
9869 	if (tcp_in_hpts(bbr->rc_tp)) {
9870 		tcp_hpts_remove(bbr->rc_tp);
9875 bbr_google_mode_on(struct tcp_bbr *bbr)
9877 	bbr->rc_use_google = 1;
9878 	bbr->rc_no_pacing = 0;
9879 	bbr->r_ctl.bbr_google_discount = bbr_google_discount;
9880 	bbr->r_use_policer = bbr_policer_detection_enabled;
9881 	bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10);
9882 	bbr->bbr_use_rack_cheat = 0;
9883 	bbr->r_ctl.rc_incr_tmrs = 0;
9884 	bbr->r_ctl.rc_inc_tcp_oh = 0;
9885 	bbr->r_ctl.rc_inc_ip_oh = 0;
9886 	bbr->r_ctl.rc_inc_enet_oh = 0;
9887 	reset_time(&bbr->r_ctl.rc_delrate,
9889 	reset_time_small(&bbr->r_ctl.rc_rttprop,
9891 	tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv));
9895 bbr_google_mode_off(struct tcp_bbr *bbr)
9897 	bbr->rc_use_google = 0;
9898 	bbr->r_ctl.bbr_google_discount = 0;
9899 	bbr->no_pacing_until = bbr_no_pacing_until;
9900 	bbr->r_use_policer = 0;
9901 	if (bbr->no_pacing_until)
9902 		bbr->rc_no_pacing = 1;
9904 		bbr->rc_no_pacing = 0;
9906 		bbr->bbr_use_rack_cheat = 1;
9908 		bbr->bbr_use_rack_cheat = 0;
9910 		bbr->r_ctl.rc_incr_tmrs = 1;
9912 		bbr->r_ctl.rc_incr_tmrs = 0;
9914 		bbr->r_ctl.rc_inc_tcp_oh = 1;
9916 		bbr->r_ctl.rc_inc_tcp_oh = 0;
9918 		bbr->r_ctl.rc_inc_ip_oh = 1;
9920 		bbr->r_ctl.rc_inc_ip_oh = 0;
9922 		bbr->r_ctl.rc_inc_enet_oh = 1;
9924 		bbr->r_ctl.rc_inc_enet_oh = 0;
9925 	bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;
9926 	reset_time(&bbr->r_ctl.rc_delrate,
9928 	reset_time_small(&bbr->r_ctl.rc_rttprop,
9930 	tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv));
9940 	struct tcp_bbr *bbr = NULL;
9955 	bbr = (struct tcp_bbr *)*ptr;
9956 	bbr->rtt_valid = 0;
9965 	TAILQ_INIT(&bbr->r_ctl.rc_map);
9966 	TAILQ_INIT(&bbr->r_ctl.rc_free);
9967 	TAILQ_INIT(&bbr->r_ctl.rc_tmap);
9968 	bbr->rc_tp = tp;
9969 	bbr->rc_inp = inp;
9970 	cts = tcp_get_usecs(&bbr->rc_tv);
9972 	bbr->rc_allow_data_af_clo = bbr_ignore_data_after_close;
9973 	bbr->r_ctl.rc_reorder_fade = bbr_reorder_fade;
9974 	bbr->rc_tlp_threshold = bbr_tlp_thresh;
9975 	bbr->r_ctl.rc_reorder_shift = bbr_reorder_thresh;
9976 	bbr->r_ctl.rc_pkt_delay = bbr_pkt_delay;
9977 	bbr->r_ctl.rc_min_to = bbr_min_to;
9978 	bbr->rc_bbr_state = BBR_STATE_STARTUP;
9979 	bbr->r_ctl.bbr_lost_at_state = 0;
9980 	bbr->r_ctl.rc_lost_at_startup = 0;
9981 	bbr->rc_all_timers_stopped = 0;
9982 	bbr->r_ctl.rc_bbr_lastbtlbw = 0;
9983 	bbr->r_ctl.rc_pkt_epoch_del = 0;
9984 	bbr->r_ctl.rc_pkt_epoch = 0;
9985 	bbr->r_ctl.rc_lowest_rtt = 0xffffffff;
9986 	bbr->r_ctl.rc_bbr_hptsi_gain = bbr_high_gain;
9987 	bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain;
9988 	bbr->r_ctl.rc_went_idle_time = cts;
9989 	bbr->rc_pacer_started = cts;
9990 	bbr->r_ctl.rc_pkt_epoch_time = cts;
9991 	bbr->r_ctl.rc_rcvtime = cts;
9992 	bbr->r_ctl.rc_bbr_state_time = cts;
9993 	bbr->r_ctl.rc_del_time = cts;
9994 	bbr->r_ctl.rc_tlp_rxt_last_time = cts;
9995 	bbr->r_ctl.last_in_probertt = cts;
9996 	bbr->skip_gain = 0;
9997 	bbr->gain_is_limited = 0;
9998 	bbr->no_pacing_until = bbr_no_pacing_until;
9999 	if (bbr->no_pacing_until)
10000 		bbr->rc_no_pacing = 1;
10002 		bbr->rc_no_pacing = 0;
10003 		bbr->rc_use_google = 1;
10004 		bbr->r_ctl.bbr_google_discount = bbr_google_discount;
10005 		bbr->r_use_policer = bbr_policer_detection_enabled;
10007 		bbr->rc_use_google = 0;
10008 		bbr->r_ctl.bbr_google_discount = 0;
10009 		bbr->r_use_policer = 0;
10012 		bbr->rc_use_ts_limit = 1;
10014 		bbr->rc_use_ts_limit = 0;
10016 		bbr->ts_can_raise = 1;
10018 		bbr->ts_can_raise = 0;
10027 	if (bbr->rc_use_google == 0)
10028 		bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;
10030 		bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10);
10031 	bbr->r_ctl.rc_min_rto_ms = bbr_rto_min_ms;
10032 	bbr->rc_max_rto_sec = bbr_rto_max_sec;
10033 	bbr->rc_init_win = bbr_def_init_win;
10035 		bbr->rc_last_options = TCP_TS_OVERHEAD;
10036 	bbr->r_ctl.rc_pace_max_segs = tp->t_maxseg - bbr->rc_last_options;
10037 	bbr->r_ctl.rc_high_rwnd = tp->snd_wnd;
10038 	bbr->r_init_rtt = 1;
10042 		bbr->bbr_hdw_pace_ena = 1;
10044 		bbr->bbr_hdw_pace_ena = 0;
10046 		bbr->bbr_init_win_cheat = 1;
10048 		bbr->bbr_init_win_cheat = 0;
10049 	bbr->r_ctl.bbr_utter_max = bbr_hptsi_utter_max;
10050 	bbr->r_ctl.rc_drain_pg = bbr_drain_gain;
10051 	bbr->r_ctl.rc_startup_pg = bbr_high_gain;
10052 	bbr->rc_loss_exit = bbr_exit_startup_at_loss;
10053 	bbr->r_ctl.bbr_rttprobe_gain_val = bbr_rttprobe_gain;
10054 	bbr->r_ctl.bbr_hptsi_per_second = bbr_hptsi_per_second;
10055 	bbr->r_ctl.bbr_hptsi_segments_delay_tar = bbr_hptsi_segments_delay_tar;
10056 	bbr->r_ctl.bbr_hptsi_segments_max = bbr_hptsi_segments_max;
10057 	bbr->r_ctl.bbr_hptsi_segments_floor = bbr_hptsi_segments_floor;
10058 	bbr->r_ctl.bbr_hptsi_bytes_min = bbr_hptsi_bytes_min;
10059 	bbr->r_ctl.bbr_cross_over = bbr_cross_over;
10060 	bbr->r_ctl.rc_rtt_shrinks = cts;
10061 	if (bbr->rc_use_google) {
10062 		setup_time_filter(&bbr->r_ctl.rc_delrate,
10065 		setup_time_filter_small(&bbr->r_ctl.rc_rttprop,
10068 		setup_time_filter(&bbr->r_ctl.rc_delrate,
10071 		setup_time_filter_small(&bbr->r_ctl.rc_rttprop,
10074 	bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_INIT, 0);
10076 		bbr->rc_use_idle_restart = 1;
10078 		bbr->rc_use_idle_restart = 0;
10079 	bbr->r_ctl.rc_bbr_cur_del_rate = 0;
10080 	bbr->r_ctl.rc_initial_hptsi_bw = bbr_initial_bw_bps;
10082 		bbr->rc_resends_use_tso = 1;
10087 		rsm = bbr_alloc(bbr);
10100 		rsm->r_delivered = bbr->r_ctl.rc_delivered;
10104 		if (bbr->r_ctl.r_app_limited_until)
10108 		TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next);
10109 		TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
10111 		if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)
10112 			rsm->r_bbr_state = bbr_state_val(bbr);
10116 	if (bbr_use_rack_resend_cheat && (bbr->rc_use_google == 0))
10117 		bbr->bbr_use_rack_cheat = 1;
10118 	if (bbr_incr_timers && (bbr->rc_use_google == 0))
10119 		bbr->r_ctl.rc_incr_tmrs = 1;
10120 	if (bbr_include_tcp_oh && (bbr->rc_use_google == 0))
10121 		bbr->r_ctl.rc_inc_tcp_oh = 1;
10122 	if (bbr_include_ip_oh && (bbr->rc_use_google == 0))
10123 		bbr->r_ctl.rc_inc_ip_oh = 1;
10124 	if (bbr_include_enet_oh && (bbr->rc_use_google == 0))
10125 		bbr->r_ctl.rc_inc_enet_oh = 1;
10127 	bbr_log_type_statechange(bbr, cts, __LINE__);
10133 		apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
10136 	bbr_log_settings_change(bbr, BBR_RECOVERY_LOWRTT);
10137 	tcp_bbr_tso_size_check(bbr, cts);
10143 	bbr_stop_all_timers(tp, bbr);
10154 	bbr_start_hpts_timer(bbr, tp, cts, 5, 0, 0);
10198 		struct tcp_bbr *bbr;
10201 		bbr = (struct tcp_bbr *)tp->t_fb_ptr;
10202 		if (bbr->r_ctl.crte)
10203 			tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp);
10204 		bbr_log_flowend(bbr);
10205 		bbr->rc_tp = NULL;
10206 		if (bbr->bbr_hdrw_pacing)
10210 		if (bbr->r_ctl.crte != NULL) {
10211 			tcp_rel_pacing_rate(bbr->r_ctl.crte, tp);
10212 			bbr->r_ctl.crte = NULL;
10214 		rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
10216 			TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);
10218 			rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
10220 		rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);
10222 			TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next);
10224 			rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);
10226 		calc = bbr->r_ctl.rc_high_rwnd - bbr->r_ctl.rc_init_rwnd;
10227 		if (calc > (bbr->r_ctl.rc_init_rwnd / 10))
10231 		bbr->r_ctl.rc_free_cnt = 0;
10240 bbr_set_state(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t win)
10244 		bbr->r_state = TCPS_SYN_SENT;
10245 		bbr->r_substate = bbr_do_syn_sent;
10248 		bbr->r_state = TCPS_SYN_RECEIVED;
10249 		bbr->r_substate = bbr_do_syn_recv;
10252 		bbr->r_ctl.rc_init_rwnd = max(win, bbr->rc_tp->snd_wnd);
10253 		bbr->r_state = TCPS_ESTABLISHED;
10254 		bbr->r_substate = bbr_do_established;
10257 		bbr->r_state = TCPS_CLOSE_WAIT;
10258 		bbr->r_substate = bbr_do_close_wait;
10261 		bbr->r_state = TCPS_FIN_WAIT_1;
10262 		bbr->r_substate = bbr_do_fin_wait_1;
10265 		bbr->r_state = TCPS_CLOSING;
10266 		bbr->r_substate = bbr_do_closing;
10269 		bbr->r_state = TCPS_LAST_ACK;
10270 		bbr->r_substate = bbr_do_lastack;
10273 		bbr->r_state = TCPS_FIN_WAIT_2;
10274 		bbr->r_substate = bbr_do_fin_wait_2;
10285 bbr_substate_change(struct tcp_bbr *bbr, uint32_t cts, int32_t line, int dolog)
10293 	old_state = bbr_state_val(bbr);
10294 	if (bbr_state_val(bbr) == BBR_SUB_LEVEL1) {
10296 		bbr->rc_hit_state_1 = 0;
10297 		if (bbr->r_ctl.bbr_smallest_srtt_this_state != 0xffffffff)
10298 			bbr->r_ctl.bbr_smallest_srtt_state2 = bbr->r_ctl.bbr_smallest_srtt_this_state;
10300 	bbr->rc_bbr_substate++;
10301 	if (bbr_state_val(bbr) == BBR_SUB_GAIN) {
10306 		if (bbr->skip_gain) {
10312 			bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_LEVEL1];
10313 		} else if (bbr->gain_is_limited &&
10314 			   bbr->bbr_hdrw_pacing &&
10315 			   bbr->r_ctl.crte) {
10324 			bw = bbr_get_bw(bbr);
10325 			rate = bbr->r_ctl.crte->rate;
10331 				bbr->r_ctl.rc_bbr_hptsi_gain = (uint16_t)gain_calc;
10333 				bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN];
10336 			bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN];
10337 		if ((bbr->rc_use_google == 0) && (bbr_gain_to_target == 0)) {
10338 			bbr->r_ctl.rc_bbr_state_atflight = cts;
10340 			bbr->r_ctl.rc_bbr_state_atflight = 0;
10341 	} else if (bbr_state_val(bbr) == BBR_SUB_DRAIN) {
10342 		bbr->rc_hit_state_1 = 1;
10343 		bbr->r_ctl.rc_exta_time_gd = 0;
10344 		bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp,
10345 						     (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
10347 			bbr->r_ctl.rc_bbr_state_atflight = 0;
10349 			bbr->r_ctl.rc_bbr_state_atflight = cts;
10350 		bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_DRAIN];
10353 		if ((old_state == BBR_SUB_DRAIN) && bbr->rc_hit_state_1) {
10355 			    (bbr->rc_use_google == 0) &&
10356 			    (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {
10357 				bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
10358 				bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10360 			if ((cts - bbr->r_ctl.rc_bbr_state_time) > bbr_get_rtt(bbr, BBR_RTT_PROP))
10361 				bbr->r_ctl.rc_exta_time_gd += ((cts - bbr->r_ctl.rc_bbr_state_time) -
10362 							       bbr_get_rtt(bbr, BBR_RTT_PROP));
10364 				bbr->r_ctl.rc_exta_time_gd = 0;
10365 			if (bbr->r_ctl.rc_exta_time_gd) {
10366 				bbr->r_ctl.rc_level_state_extra = bbr->r_ctl.rc_exta_time_gd;
10368 				bbr->r_ctl.rc_level_state_extra /= 7;
10369 				if (bbr_rand_ot && bbr->r_ctl.rc_level_state_extra) {
10371 					bbr_randomize_extra_state_time(bbr);
10375 		bbr->r_ctl.rc_bbr_state_atflight = max(1, cts);
10376 		bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[bbr_state_val(bbr)];
10378 	if (bbr->rc_use_google) {
10379 		bbr->r_ctl.rc_bbr_state_atflight = max(1, cts);
10381 	bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
10382 	bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain;
10384 		bbr_log_type_statechange(bbr, cts, line);
10386 	if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
10389 		time_in = cts - bbr->r_ctl.rc_bbr_state_time;
10390 		if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {
10393 			counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
10396 	bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff;
10397 	bbr_set_state_target(bbr, __LINE__);
10399 	    (bbr->rc_use_google == 0) &&
10400 	    (bbr_state_val(bbr) == BBR_SUB_DRAIN)) {
10402 		bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
10403 		bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
10406 			bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered +
10407 							  ctf_flight_size(bbr->rc_tp,
10408 							      (bbr->r_ctl.rc_sacked +
10409 							       bbr->r_ctl.rc_lost_bytes)));
10411 		bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10413 	if (bbr->rc_lt_use_bw) {
10415 		bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
10418 	if (bbr->rc_use_google)
10419 		tcp_bbr_tso_size_check(bbr, cts);
10420 	bbr->r_ctl.gain_epoch = cts;
10421 	bbr->r_ctl.rc_bbr_state_time = cts;
10422 	bbr->r_ctl.substate_pe = bbr->r_ctl.rc_pkt_epoch;
10426 bbr_set_probebw_google_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses)
10428 	if ((bbr_state_val(bbr) == BBR_SUB_DRAIN) &&
10430 	    (bbr->r_ctl.rc_flight_at_input <= bbr->r_ctl.rc_target_at_state)) {
10434 	if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time)) {
10437 	if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_get_rtt(bbr, BBR_RTT_PROP)) {
10444 	if (bbr_state_val(bbr) == BBR_SUB_GAIN) {
10454 		if (bbr->r_ctl.rc_target_at_state > bbr->r_ctl.rc_flight_at_input) {
10460 	bbr_substate_change(bbr, cts, __LINE__, 1);
10464 bbr_set_probebw_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses)
10468 	if (bbr->rc_use_google) {
10469 		bbr_set_probebw_google_gains(bbr, cts, losses);
10481 		bbr_cur_cycle_time = bbr_get_rtt(bbr, BBR_RTT_PKTRTT);
10483 		bbr_cur_cycle_time = bbr_get_rtt(bbr, BBR_RTT_PROP);
10485 	if (bbr->r_ctl.rc_bbr_state_atflight == 0) {
10486 		if (bbr_state_val(bbr) == BBR_SUB_DRAIN) {
10487 			flight = ctf_flight_size(bbr->rc_tp,
10488 				     (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
10489 			if (bbr_sub_drain_slam_cwnd && bbr->rc_hit_state_1) {
10491 				if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state) {
10492 					bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
10493 					bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10497 					bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered + flight);
10500 			if (TSTMP_GT(cts, bbr->r_ctl.gain_epoch) &&
10501 			    (((cts - bbr->r_ctl.gain_epoch) > bbr_get_rtt(bbr, BBR_RTT_PROP)) ||
10502 			     (flight >= bbr->r_ctl.flightsize_at_drain))) {
10510 				bbr->r_ctl.flightsize_at_drain = flight;
10515 					bbr->r_ctl.rc_bbr_hptsi_gain *= bbr_drain_drop_mul;
10516 					bbr->r_ctl.rc_bbr_hptsi_gain /= bbr_drain_drop_div;
10519 					bbr->r_ctl.rc_bbr_hptsi_gain *= 4;
10520 					bbr->r_ctl.rc_bbr_hptsi_gain /= 5;
10522 				if (bbr->r_ctl.rc_bbr_hptsi_gain <= bbr_drain_floor) {
10524 					bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1);
10526 				bbr_log_exit_gain(bbr, cts, 4);
10531 				bbr->r_ctl.gain_epoch = cts;
10533 			if (flight <= bbr->r_ctl.rc_target_at_state) {
10535 				    (bbr->rc_use_google == 0) &&
10536 				    (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {
10537 					bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
10538 					bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10540 				bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);
10541 				bbr_log_exit_gain(bbr, cts, 3);
10545 			if (bbr->r_ctl.rc_lost > bbr->r_ctl.bbr_lost_at_state) {
10546 				bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);
10549 			if ((ctf_outstanding(bbr->rc_tp) >= bbr->r_ctl.rc_target_at_state) ||
10550 			    ((ctf_outstanding(bbr->rc_tp) +  bbr->rc_tp->t_maxseg - 1) >=
10551 			     bbr->rc_tp->snd_wnd)) {
10552 				bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);
10553 				bbr_log_exit_gain(bbr, cts, 2);
10568 	if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time))
10570 	if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_cur_cycle_time) {
10574 	if (bbr->r_ctl.rc_level_state_extra &&
10575 	    (bbr_state_val(bbr) > BBR_SUB_DRAIN) &&
10576 	    ((cts - bbr->r_ctl.rc_bbr_state_time) <
10577 	     (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) {
10582 	    bbr->r_ctl.rc_level_state_extra &&
10583 	    (bbr_state_val(bbr) == BBR_SUB_GAIN) &&
10584 	    ((cts - bbr->r_ctl.rc_bbr_state_time) <
10585 	     (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) {
10589 	bbr_substate_change(bbr, cts, __LINE__, 1);
10593 bbr_get_a_state_target(struct tcp_bbr *bbr, uint32_t gain)
10597 	if (bbr->rc_use_google) {
10599 		tar = bbr_get_target_cwnd(bbr, bbr_get_bw(bbr), gain);
10601 		mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options),
10602 			  bbr->r_ctl.rc_pace_max_segs);
10604 		tar = roundup(bbr_get_raw_target_cwnd(bbr, bbr_get_bw(bbr),
10607 		if (tar < get_min_cwnd(bbr))
10608 			return (get_min_cwnd(bbr));
10614 bbr_set_state_target(struct tcp_bbr *bbr, int line)
10618 	if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&
10619 	    ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) {
10621 		tar = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
10625 		if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&
10626 		    (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT)) {
10628 			tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain);
10630 		} else if ((bbr_target_is_bbunit) || bbr->rc_use_google) {
10635 			tar = bbr_get_a_state_target(bbr, BBR_UNIT);
10643 			if (bbr->r_ctl.rc_bbr_hptsi_gain < bbr_hptsi_gain[BBR_SUB_DRAIN])  {
10644 				tar = bbr_get_a_state_target(bbr, bbr_hptsi_gain[BBR_SUB_DRAIN]);
10647 				tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain);
10652 	bbr_log_set_of_state_target(bbr, tar, line, meth);
10653 	bbr->r_ctl.rc_target_at_state = tar;
10657 bbr_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
10662 	bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
10663 	bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp,
10664 					     (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
10665 	bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.flightsize_at_drain
10666 					  + bbr->r_ctl.rc_delivered);
10668 	if (bbr->rc_use_google || bbr_probertt_sets_rtt)
10669 		bbr->rc_prtt_set_ts = 1;
10670 	if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
10671 		time_in = cts - bbr->r_ctl.rc_bbr_state_time;
10672 		counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
10674 	bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_ENTERPROBE, 0);
10675 	bbr->r_ctl.rc_rtt_shrinks = cts;
10676 	bbr->r_ctl.last_in_probertt = cts;
10677 	bbr->r_ctl.rc_probertt_srttchktim = cts;
10678 	bbr->r_ctl.rc_bbr_state_time = cts;
10679 	bbr->rc_bbr_state = BBR_STATE_PROBE_RTT;
10683 	    bbr->rc_hit_state_1 &&
10684 	    (bbr->rc_use_google == 0) &&
10685 	    (bbr_state_val(bbr) == BBR_SUB_DRAIN)) {
10686 		if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_saved_cwnd)
10687 			bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
10689 		bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
10691 	bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
10692 	if ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google){
10694 		bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
10695 		bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10696 		bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
10697 		bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
10698 		bbr_log_set_of_state_target(bbr, bbr->rc_tp->snd_cwnd, __LINE__, 6);
10699 		bbr->r_ctl.rc_target_at_state = bbr->rc_tp->snd_cwnd;
10706 		bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val;
10707 		bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
10708 		bbr_set_state_target(bbr, __LINE__);
10710 		    (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
10711 			bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
10712 			bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10715 	if (ctf_flight_size(bbr->rc_tp,
10716 		(bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <=
10717 	    bbr->r_ctl.rc_target_at_state) {
10719 		bbr->r_ctl.rc_bbr_enters_probertt = cts;
10722 		bbr->r_ctl.rc_bbr_enters_probertt = 0;
10724 	bbr->r_ctl.rc_pe_of_prtt = bbr->r_ctl.rc_pkt_epoch;
10726 	bbr_log_exit_gain(bbr, cts, 0);
10727 	bbr_log_type_statechange(bbr, cts, line);
10731 bbr_check_probe_rtt_limits(struct tcp_bbr *bbr, uint32_t cts)
10742 	    (bbr->rc_use_google == 0)) {
10747 		baseval = (bbr_get_rtt(bbr, BBR_RTT_PROP) * (BBR_SUBSTATE_COUNT + 1));
10761 		if (cur_rttp > 	bbr->r_ctl.rc_probertt_int) {
10762 			bbr->r_ctl.rc_probertt_int = cur_rttp;
10763 			reset_time_small(&bbr->r_ctl.rc_rttprop, newval);
10770 			if (bbr->r_ctl.rc_probertt_int > bbr_rtt_probe_limit) {
10776 					bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;
10777 					reset_time_small(&bbr->r_ctl.rc_rttprop,
10786 					if (cur_rttp < bbr->r_ctl.rc_probertt_int) {
10788 						bbr->r_ctl.rc_probertt_int = cur_rttp;
10789 						reset_time_small(&bbr->r_ctl.rc_rttprop, newval);
10796 			bbr_log_rtt_shrinks(bbr, cts, cur_rttp, newval, __LINE__, BBR_RTTS_RESETS_VALUES, val);
10801 bbr_exit_probe_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
10805 	if (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd) {
10806 		tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
10807 		bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10809 	bbr_log_exit_gain(bbr, cts, 1);
10810 	bbr->rc_hit_state_1 = 0;
10811 	bbr->r_ctl.rc_rtt_shrinks = cts;
10812 	bbr->r_ctl.last_in_probertt = cts;
10813 	bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_RTTPROBE, 0);
10814 	bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
10815 	bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp,
10816 					      (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
10817 					  bbr->r_ctl.rc_delivered);
10818 	if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
10821 		time_in = cts - bbr->r_ctl.rc_bbr_state_time;
10822 		counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
10824 	if (bbr->rc_filled_pipe) {
10826 		bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
10827 		bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
10828 		bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain;
10829 		bbr_substate_change(bbr, cts, __LINE__, 0);
10830 		bbr_log_type_statechange(bbr, cts, __LINE__);
10833 		bbr->rc_bbr_state = BBR_STATE_STARTUP;
10834 		bbr->r_ctl.rc_bbr_state_time = cts;
10842 		bbr->r_ctl.rc_bbr_last_startup_epoch += (bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_pe_of_prtt);
10843 		bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
10845 		if (bbr->r_ctl.rc_lost &&
10847 		    (bbr->rc_use_google == 0))
10848 			bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower;
10850 			bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;
10851 		bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;
10853 		bbr_set_state_target(bbr, __LINE__);
10854 		bbr_log_type_statechange(bbr, cts, __LINE__);
10855 		bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
10856 		    bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 0);
10858 	bbr_check_probe_rtt_limits(bbr, cts);
10862 bbr_should_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts)
10864 	if ((bbr->rc_past_init_win == 1) &&
10865 	    (bbr->rc_in_persist == 0) &&
10866 	    (bbr_calc_time(cts, bbr->r_ctl.rc_rtt_shrinks) >= bbr->r_ctl.rc_probertt_int)) {
10870 	    (bbr->rc_in_persist == 0) &&
10871 	    (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) &&
10872 	    ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) {
10879 bbr_google_startup(struct tcp_bbr *bbr, uint32_t cts, int32_t  pkt_epoch)
10888 	btlbw = bbr_get_full_bw(bbr);
10889 	gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *
10890 		 (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;
10892 		bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch;
10893 		bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
10894 				      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3);
10895 		bbr->r_ctl.rc_bbr_lastbtlbw = btlbw;
10897 	if ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS)
10899 	bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
10900 			      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8);
10905 bbr_state_startup(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch, int32_t pkt_epoch)
10912 	if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) &&
10913 	    (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {
10919 		bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_WASIDLE, 0);
10920 		bbr_set_reduced_rtt(bbr, cts, __LINE__);
10922 	if (bbr_should_enter_probe_rtt(bbr, cts)) {
10923 		bbr_enter_probe_rtt(bbr, cts, __LINE__);
10926 	if (bbr->rc_use_google)
10927 		return (bbr_google_startup(bbr, cts,  pkt_epoch));
10929 	if ((bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) &&
10932 		bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower;
10946 		if (bbr->r_ctl.rc_pkt_epoch_rtt &&
10947 		    bbr->r_ctl.startup_last_srtt &&
10948 		    (bbr->r_ctl.rc_pkt_epoch_rtt > bbr->r_ctl.startup_last_srtt)) {
10949 			delta = bbr->r_ctl.rc_pkt_epoch_rtt - bbr->r_ctl.startup_last_srtt;
10950 			rtt_gain = (delta * 100) / bbr->r_ctl.startup_last_srtt;
10953 		if ((bbr->r_ctl.startup_last_srtt == 0)  ||
10954 		    (bbr->r_ctl.rc_pkt_epoch_rtt < bbr->r_ctl.startup_last_srtt))
10956 			bbr->r_ctl.startup_last_srtt = bbr->r_ctl.rc_pkt_epoch_rtt;
10958 		if ((bbr->r_ctl.rc_lost == 0) &&
10965 			if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch)
10966 				bbr->r_ctl.rc_bbr_last_startup_epoch++;
10967 			bbr_log_startup_event(bbr, cts, rtt_gain,
10968 					      delta, bbr->r_ctl.startup_last_srtt, 10);
10972 	if ((bbr->r_ctl.r_measurement_count == bbr->r_ctl.last_startup_measure) &&
10973 	    (bbr->r_ctl.rc_lost_at_startup == bbr->r_ctl.rc_lost) &&
10974 	    (!IN_RECOVERY(bbr->rc_tp->t_flags))) {
10981 		if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch)
10982 			bbr->r_ctl.rc_bbr_last_startup_epoch++;
10983 		bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
10984 				      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 9);
10988 	if (bbr->r_ctl.rc_lost_at_startup > bbr->r_ctl.rc_lost)
10989 		bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
10990 	bbr->r_ctl.last_startup_measure = bbr->r_ctl.r_measurement_count;
10991 	btlbw = bbr_get_full_bw(bbr);
10992 	if (bbr->r_ctl.rc_bbr_hptsi_gain == bbr_startup_lower)
10993 		gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *
10994 			 (uint64_t)bbr_low_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;
10996 		gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *
10997 			 (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;
10999 	if (btlbw > bbr->r_ctl.rc_bbr_lastbtlbw)
11000 		bbr->r_ctl.rc_bbr_lastbtlbw = btlbw;
11002 		bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch;
11004 		bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
11005 		bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11006 				      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3);
11008 	if ((bbr->rc_loss_exit &&
11009 	     (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) &&
11010 	     (bbr->r_ctl.rc_pkt_epoch_loss_rate > bbr_startup_loss_thresh)) &&
11011 	    ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS)) {
11019 		if ((ctf_flight_size(bbr->rc_tp,
11020 			 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
11021 		     (2 * max(bbr->r_ctl.rc_pace_max_segs, bbr->rc_tp->t_maxseg))) <= bbr->rc_tp->snd_wnd) {
11023 			bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11024 					      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 4);
11027 			bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
11028 			bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11029 					      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 5);
11032 		bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
11033 	if (((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS) ||
11039 	bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11040 			      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8);
11045 bbr_state_change(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch, int32_t pkt_epoch, uint32_t losses)
11051 	if ((bbr->rc_bbr_state != BBR_STATE_STARTUP) &&
11052 	    (bbr->rc_bbr_state != BBR_STATE_DRAIN) &&
11053 	    (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) &&
11054 	    (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) &&
11055 	    (bbr->rc_bbr_state != BBR_STATE_PROBE_BW)) {
11057 		panic("Unknown BBR state %d?\n", bbr->rc_bbr_state);
11060 	if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
11062 		if (bbr_state_startup(bbr, cts, epoch, pkt_epoch)) {
11065 			bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11066 					      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 6);
11067 			bbr->rc_filled_pipe = 1;
11068 			bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
11069 			if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
11070 				time_in = cts - bbr->r_ctl.rc_bbr_state_time;
11071 				counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
11074 			if (bbr->rc_no_pacing)
11075 				bbr->rc_no_pacing = 0;
11076 			bbr->r_ctl.rc_bbr_state_time = cts;
11077 			bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_drain_pg;
11078 			bbr->rc_bbr_state = BBR_STATE_DRAIN;
11079 			bbr_set_state_target(bbr, __LINE__);
11080 			if ((bbr->rc_use_google == 0) &&
11083 				bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
11084 				bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
11085 				bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11087 			bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain;
11088 			bbr_log_type_statechange(bbr, cts, __LINE__);
11089 			if (ctf_flight_size(bbr->rc_tp,
11090 			        (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <=
11091 			    bbr->r_ctl.rc_target_at_state) {
11096 				bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
11097 				bbr_substate_change(bbr, cts, __LINE__, 0);
11098 				bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
11099 				bbr_log_type_statechange(bbr, cts, __LINE__);
11102 	} else if (bbr->rc_bbr_state == BBR_STATE_IDLE_EXIT) {
11106 		tp = bbr->rc_tp;
11108 			      (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11109 		if (inflight >= bbr->r_ctl.rc_target_at_state) {
11111 			bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
11112 			bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
11113 			bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
11114 			bbr_set_state_target(bbr, __LINE__);
11119 			bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff;
11120 			bbr->rc_bbr_substate = BBR_SUB_LEVEL6;
11121 			bbr_substate_change(bbr, cts, __LINE__, 1);
11123 	} else if (bbr->rc_bbr_state == BBR_STATE_DRAIN) {
11128 		tp = bbr->rc_tp;
11130 			      (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11131 		if ((bbr->rc_use_google == 0) &&
11133 		    (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
11138 			bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
11139 			bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11141 		if (inflight <= bbr->r_ctl.rc_target_at_state) {
11143 			bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
11144 			bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
11145 			if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
11148 				time_in = cts - bbr->r_ctl.rc_bbr_state_time;
11149 				counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
11151 			if ((bbr->rc_use_google == 0) &&
11153 			    (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {
11155 				tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
11156 				bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11159 			bbr->r_ctl.rc_rtt_shrinks = cts;
11160 			bbr->r_ctl.last_in_probertt = cts;
11161 			bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_LEAVE_DRAIN, 0);
11163 			bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
11164 			bbr_substate_change(bbr, cts, __LINE__, 0);
11165 			bbr_log_type_statechange(bbr, cts, __LINE__);
11167 	} else if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) {
11170 		flight = ctf_flight_size(bbr->rc_tp,
11171 			     (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11172 		bbr->r_ctl.r_app_limited_until = (flight + bbr->r_ctl.rc_delivered);
11173 		if (((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google) &&
11174 		    (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
11178 			bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
11179 			bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11181 			   (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
11183 			bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
11184 			bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11186 		if (bbr->r_ctl.rc_bbr_enters_probertt == 0) {
11188 			if (flight <= bbr->r_ctl.rc_target_at_state) {
11189 				bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_REACHTAR, 0);
11190 				bbr->r_ctl.rc_bbr_enters_probertt = cts;
11192 				if (bbr->r_ctl.rc_bbr_enters_probertt == 0)
11193 					bbr->r_ctl.rc_bbr_enters_probertt = 1;
11194 				if (bbr->rc_use_google == 0) {
11199 					if (bbr->r_ctl.bbr_rttprobe_gain_val)
11200 						bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val;
11202 						bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
11205 			if ((bbr->r_ctl.rc_bbr_enters_probertt == 0) &&
11206 			    (bbr->rc_use_google == 0) &&
11207 			    bbr->r_ctl.bbr_rttprobe_gain_val &&
11208 			    (((cts - bbr->r_ctl.rc_probertt_srttchktim) > bbr_get_rtt(bbr, bbr_drain_rtt)) ||
11209 			     (flight >= bbr->r_ctl.flightsize_at_drain))) {
11220 				bbr->r_ctl.flightsize_at_drain = flight;
11221 				bbr->r_ctl.rc_probertt_srttchktim = cts;
11222 				red = max((bbr->r_ctl.bbr_rttprobe_gain_val / 10), 1);
11223 				if ((bbr->r_ctl.rc_bbr_hptsi_gain - red) > max(bbr_drain_floor, 1)) {
11225 					bbr->r_ctl.rc_bbr_hptsi_gain -= red;
11226 					bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_SHRINK_PG, 0);
11227 				} else if (bbr->r_ctl.rc_bbr_hptsi_gain > max(bbr_drain_floor, 1)) {
11229 					bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1);
11230 					bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_SHRINK_PG_FINAL, 0);
11233 					bbr->r_ctl.rc_bbr_hptsi_gain = max((bbr_drain_floor-1), 1);
11237 		if (bbr->r_ctl.rc_bbr_enters_probertt &&
11238 		    (TSTMP_GT(cts, bbr->r_ctl.rc_bbr_enters_probertt)) &&
11239 		    ((cts - bbr->r_ctl.rc_bbr_enters_probertt) >= bbr_rtt_probe_time)) {
11241 			bbr_exit_probe_rtt(bbr->rc_tp, bbr, cts);
11243 	} else if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {
11244 		if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) &&
11245 		    (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {
11251 			bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_WASIDLE, 0);
11252 			bbr_set_reduced_rtt(bbr, cts, __LINE__);
11254 		if (bbr_should_enter_probe_rtt(bbr, cts)) {
11255 			bbr_enter_probe_rtt(bbr, cts, __LINE__);
11257 			bbr_set_probebw_gains(bbr, cts, losses);
11263 bbr_check_bbr_for_state(struct tcp_bbr *bbr, uint32_t cts, int32_t line, uint32_t losses)
11267 	if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP)) {
11268 		bbr_set_epoch(bbr, cts, line);
11272 	bbr_state_change(bbr, cts, epoch, bbr->rc_is_pkt_epoch_now, losses);
11286 	struct tcp_bbr *bbr;
11295 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
11297 	kern_prefetch(bbr, &prev_state);
11328 		bbr->rc_tv.tv_sec = ts.tv_sec;
11329 		bbr->rc_tv.tv_usec = ts.tv_nsec / 1000;
11330 		bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usectick(&bbr->rc_tv);
11336 		bbr->rc_tv.tv_sec = ts.tv_sec;
11337 		bbr->rc_tv.tv_usec = ts.tv_nsec / 1000;
11338 		bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usectick(&bbr->rc_tv);
11343 		bbr->r_ctl.rc_rcvtime = lcts = cts = tcp_get_usecs(&bbr->rc_tv);
11379 		if (TSTMP_GT(to.to_tsecr, tcp_tv_to_mssectick(&bbr->rc_tv)))
11386 	if (bbr->r_state == 0) {
11394 		if (bbr->rc_inp == NULL) {
11395 			bbr->rc_inp = inp;
11417 				tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
11456 		bbr->r_is_v6 = (inp->inp_vflag & INP_IPV6) != 0;
11458 		sack_filter_clear(&bbr->r_ctl.bbr_sf, th->th_ack);
11474 	if (bbr->r_state != tp->t_state)
11475 		bbr_set_state(tp, bbr, tiwin);
11477 	if (SEQ_GT(th->th_ack, tp->snd_una) && (rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map)) != NULL)
11479 	prev_state = bbr->r_state;
11480 	bbr->rc_ack_was_delayed = 0;
11481 	lost = bbr->r_ctl.rc_lost;
11482 	bbr->rc_is_pkt_epoch_now = 0;
11487 			bbr->r_ctl.rc_ack_hdwr_delay = lcts - cts;
11488 			bbr->rc_ack_was_delayed = 1;
11489 			if (TSTMP_GT(bbr->r_ctl.rc_ack_hdwr_delay,
11490 				     bbr->r_ctl.highest_hdwr_delay))
11491 				bbr->r_ctl.highest_hdwr_delay = bbr->r_ctl.rc_ack_hdwr_delay;
11493 			bbr->r_ctl.rc_ack_hdwr_delay = 0;
11494 			bbr->rc_ack_was_delayed = 0;
11497 		bbr->r_ctl.rc_ack_hdwr_delay = 0;
11498 		bbr->rc_ack_was_delayed = 0;
11500 	bbr_log_ack_event(bbr, th, &to, tlen, nsegs, cts, nxt_pkt, m);
11516 	if (tiwin > bbr->r_ctl.rc_high_rwnd)
11517 		bbr->r_ctl.rc_high_rwnd = tiwin;
11518 	bbr->r_ctl.rc_flight_at_input = ctf_flight_size(tp,
11519 					    (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11520 	bbr->rtt_valid = 0;
11522 		bbr->rc_ts_valid = 1;
11523 		bbr->r_ctl.last_inbound_ts = to.to_tsval;
11525 		bbr->rc_ts_valid = 0;
11526 		bbr->r_ctl.last_inbound_ts = 0;
11528 	retval = (*bbr->r_substate) (m, th, so,
11541 		tcp_bbr_xmit_timer_commit(bbr, tp, cts);
11542 		if (bbr->rc_is_pkt_epoch_now)
11543 			bbr_set_pktepoch(bbr, cts, __LINE__);
11544 		bbr_check_bbr_for_state(bbr, cts, __LINE__, (bbr->r_ctl.rc_lost - lost));
11546 			if ((bbr->r_wanted_output != 0) ||
11549 				bbr->rc_output_starts_timer = 0;
11554 				bbr_start_hpts_timer(bbr, tp, cts, 6, 0, 0);
11557 		    ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) == 0) &&
11560 		     ((V_tcp_always_keepalive || bbr->rc_inp->inp_socket->so_options & SO_KEEPALIVE) &&
11569 			    (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
11578 					if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
11579 					    (TSTMP_GT(lcts, bbr->rc_pacer_started))) {
11582 						del = lcts - bbr->rc_pacer_started;
11583 						if (bbr->r_ctl.rc_last_delay_val > del) {
11585 							bbr->r_ctl.rc_last_delay_val -= del;
11586 							bbr->rc_pacer_started = lcts;
11589 							bbr->r_ctl.rc_last_delay_val = 0;
11596 				bbr_start_hpts_timer(bbr, tp, cts, 8, bbr->r_ctl.rc_last_delay_val,
11599 		} else if ((bbr->rc_output_starts_timer == 0) && (nxt_pkt == 0)) {
11601 			bbr_timer_audit(tp, bbr, lcts, &so->so_snd);
11607 		if (bbr->r_state != tp->t_state)
11608 			bbr_set_state(tp, bbr, tiwin);
11610 		bbr_log_doseg_done(bbr, cts, nxt_pkt, did_out);
11612 			bbr->r_wanted_output = 0;
11650 bbr_what_can_we_send(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t sendwin,
11661 		flight = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11687 bbr_do_send_accounting(struct tcpcb *tp, struct tcp_bbr *bbr, struct bbr_sendmap *rsm, int32_t len, int32_t error)
11715 		if (bbr->rc_bbr_state != BBR_STATE_PROBE_BW) {
11717 			counter_u64_add(bbr_state_resend[bbr->rc_bbr_state], len);
11725 			counter_u64_add(bbr_state_resend[(bbr_state_val(bbr) + 5)], len);
11749 bbr_cwnd_limiting(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t in_level)
11751 	if (bbr->rc_filled_pipe && bbr_target_cwnd_mult_limit && (bbr->rc_use_google == 0)) {
11759 		target = bbr_get_target_cwnd(bbr, bbr_get_bw(bbr), BBR_UNIT);
11764 		bbr_log_type_cwndupd(bbr, 0, 0, 0, 10, 0, 0, __LINE__);
11831 	struct tcp_bbr *bbr;
11870 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
11872 	memcpy(&bbr->rc_tv, tv, sizeof(struct timeval));
11873 	cts = tcp_tv_to_usectick(&bbr->rc_tv);
11874 	inp = bbr->rc_inp;
11884 	maxseg = tp->t_maxseg - bbr->rc_last_options;
11885 	if (bbr_minseg(bbr) < maxseg) {
11886 		tcp_bbr_tso_size_check(bbr, cts);
11889 	pace_max_segs = bbr->r_ctl.rc_pace_max_segs;
11898 	if (bbr->r_state) {
11900 		isipv6 = bbr->r_is_v6;
11905 	if (((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) &&
11936 		bbr_timer_cancel(bbr, __LINE__, cts);
11938 	if (bbr->r_ctl.rc_last_delay_val) {
11940 		if (SEQ_GT(cts, bbr->rc_pacer_started))
11941 			delay_calc = cts - bbr->rc_pacer_started;
11942 		if (delay_calc >= bbr->r_ctl.rc_last_delay_val)
11943 			delay_calc -= bbr->r_ctl.rc_last_delay_val;
11948 	if ((bbr->r_timer_override) ||
11954 		if ((bbr->r_ctl.rc_last_delay_val) &&
11955 		    (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
11964 			bbr->r_ctl.rc_last_delay_val = 0;
11967 			bbr->r_ctl.rc_last_delay_val = 0;
11979 	bbr->rc_cwnd_limited = 0;
11980 	if (bbr->r_ctl.rc_last_delay_val) {
11982 		if (SEQ_GT(cts, bbr->rc_pacer_started))
11983 			delay_calc = cts - bbr->rc_pacer_started;
11986 		if (delay_calc >= bbr->r_ctl.rc_last_delay_val)
11988 			delay_calc -= bbr->r_ctl.rc_last_delay_val;
11996 			bbr->r_ctl.rc_agg_early += (bbr->r_ctl.rc_last_delay_val - delay_calc);
11997 			bbr->r_agg_early_set = 1;
11998 			if (bbr->r_ctl.rc_hptsi_agg_delay) {
11999 				if (bbr->r_ctl.rc_hptsi_agg_delay >= bbr->r_ctl.rc_agg_early) {
12001 					bbr->r_ctl.rc_hptsi_agg_delay -= bbr->r_ctl.rc_agg_early;
12002 					bbr->r_agg_early_set = 0;
12003 					bbr->r_ctl.rc_agg_early = 0;
12005 					bbr->r_ctl.rc_agg_early -= bbr->r_ctl.rc_hptsi_agg_delay;
12006 					bbr->r_ctl.rc_hptsi_agg_delay = 0;
12009 			merged_val = bbr->rc_pacer_started;
12011 			merged_val |= bbr->r_ctl.rc_last_delay_val;
12012 			bbr_log_pacing_delay_calc(bbr, hpts_calling,
12013 						 bbr->r_ctl.rc_agg_early, cts, delay_calc, merged_val,
12014 						 bbr->r_agg_early_set, 3);
12015 			bbr->r_ctl.rc_last_delay_val = 0;
12021 		if (bbr->r_agg_early_set)
12022 			bbr->r_ctl.rc_agg_early = 0;
12023 		bbr->r_agg_early_set = 0;
12032 		if ((bbr->r_ctl.rc_hptsi_agg_delay + delay_calc) < bbr->r_ctl.rc_hptsi_agg_delay)
12033 			bbr->r_ctl.rc_hptsi_agg_delay = 0xffffffff;
12035 			bbr->r_ctl.rc_hptsi_agg_delay += delay_calc;
12039 	    (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) &&
12046 		bbr_restart_after_idle(bbr,
12047 				       cts, bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time));
12055 	if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
12058 		retval = bbr_process_timers(tp, bbr, cts, hpts_calling);
12068 	bbr->rc_tp->t_flags2 &= ~TF2_MBUF_QUEUE_READY;
12070 	    (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
12071 		bbr->r_ctl.rc_last_delay_val = 0;
12073 	bbr->r_timer_override = 0;
12074 	bbr->r_wanted_output = 0;
12092 	if (bbr->rc_use_google == 0)
12093 		bbr_check_bbr_for_state(bbr, cts, __LINE__, 0);
12122 	while (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) {
12124 		rsm = bbr_alloc(bbr);
12134 		TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next);
12135 		bbr->r_ctl.rc_free_cnt++;
12139 	if (bbr->r_ctl.rc_resend == NULL) {
12141 		bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);
12142 		if (bbr->r_ctl.rc_resend) {
12146 			bbr_cong_signal(tp, NULL, CC_NDUPACK, bbr->r_ctl.rc_resend);
12149 	if (bbr->r_ctl.rc_resend) {
12150 		rsm = bbr->r_ctl.rc_resend;
12156 		bbr->r_ctl.rc_resend = NULL;
12159 			panic("Huh, tp:%p bbr:%p rsm:%p start:%u < snd_una:%u\n",
12160 			    tp, bbr, rsm, rsm->r_start, tp->snd_una);
12190 		if ((bbr->rc_resends_use_tso == 0) &&
12207 	} else if (bbr->r_ctl.rc_tlp_send) {
12212 		rsm = bbr->r_ctl.rc_tlp_send;
12213 		bbr->r_ctl.rc_tlp_send = NULL;
12216 		if ((bbr->rc_resends_use_tso == 0) && (len > maxseg))
12222 			    tp, bbr, tp->snd_una, rsm, rsm->r_start);
12238 	    (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {
12240 		if (!bbr->alloc_limit_reported) {
12241 			bbr->alloc_limit_reported = 1;
12248 		panic("tp:%p bbr:%p rsm:%p sb_offset:%u len:%u",
12249 		    tp, bbr, rsm, sb_offset, len);
12279 		end_rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext);
12308 		if (bbr->rc_tlp_new_data) {
12323 			bbr->rc_tlp_new_data = 0;
12325 			len = bbr_what_can_we_send(tp, bbr, sendwin, avail, sb_offset, cts);
12327 			    (bbr->rc_in_persist == 0) &&
12341 			if (bbr->rc_in_persist) {
12348 				rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
12353 					if ((bbr->rc_resends_use_tso == 0) && (len > maxseg))
12466 			bbr_enter_persist(tp, bbr, cts, __LINE__);
12470 		   (len < bbr->r_ctl.rc_pace_max_segs)) {
12476 		if ((tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
12488 				bbr_enter_persist(tp, bbr, cts, __LINE__);
12489 		} else if ((tp->snd_cwnd >= bbr->r_ctl.rc_pace_max_segs) &&
12490 			   (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12491 						 bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) &&
12493 			   (len < bbr_minseg(bbr))) {
12504 			bbr->rc_cwnd_limited = 1;
12507 			    min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
12508 			   (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12509 						 bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) &&
12529 	if (bbr->rc_in_persist &&
12532 	    (len < min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs))) {
12543 		if (sbleft >= min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs)) {
12704 		slot = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0);
12705 	if (bbr->rc_no_pacing)
12708 		if ((ctf_outstanding(tp) + min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) >=
12712 			bbr_cwnd_limiting(tp, bbr, ctf_outstanding(tp));
12713 			if ((bbr->rc_in_persist == 0) &&
12718 				bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
12723 			bbr_cwnd_limiting(tp, bbr, ctf_outstanding(tp));
12724 		} else if ((ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12725 						 bbr->r_ctl.rc_lost_bytes)) + p_maxseg) >= tp->snd_cwnd) {
12728 			bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12729 									bbr->r_ctl.rc_lost_bytes)));
12730 			bbr->rc_cwnd_limited = 1;
12734 			bbr_cwnd_limiting(tp, bbr, ctf_outstanding(tp));
12736 		bbr->r_ctl.rc_hptsi_agg_delay = 0;
12737 		bbr->r_agg_early_set = 0;
12738 		bbr->r_ctl.rc_agg_early = 0;
12739 		bbr->r_ctl.rc_last_delay_val = 0;
12740 	} else if (bbr->rc_use_google == 0)
12741 		bbr_check_bbr_for_state(bbr, cts, __LINE__, 0);
12748 		bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12749 								       bbr->r_ctl.rc_lost_bytes)) + bbr->r_ctl.rc_delivered);
12754 	bbr->r_ctl.rc_last_delay_val = 0;
12755 	bbr->rc_output_starts_timer = 1;
12756 	bbr_start_hpts_timer(bbr, tp, cts, 9, slot, tot_len);
12757 	bbr_log_type_just_return(bbr, cts, tot_len, hpts_calling, app_limited, p_maxseg, len);
12773 		bbr->rc_tlp_in_progress = 0;
12774 		bbr->rc_tlp_rtx_out = 0;
12779 		bbr->rc_tlp_in_progress = 1;
12781 	bbr_timer_cancel(bbr, __LINE__, cts);
12812 		    (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {
12818 			bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_WASIDLE, 0);
12819 			bbr_set_reduced_rtt(bbr, cts, __LINE__);
12888 			to.to_tsval = 	tcp_tv_to_mssectick(&bbr->rc_tv) + tp->ts_offset;
12896 			tp->rfbuf_ts = 	tcp_tv_to_mssectick(&bbr->rc_tv);
12944 	if (bbr->rc_last_options != local_options) {
12949 		bbr->rc_last_options = local_options;
13046 			panic("RSM:%p TP:%p bbr:%p start:%u is < snd_una:%u",
13047 			    rsm, tp, bbr, rsm->r_start, tp->snd_una);
13085 			bbr_log_enobuf_jmp(bbr, len, cts, __LINE__, len, 0, 0);
13100 				panic("tp:%p bbr:%p len:%u sb_offset:%u sbavail:%u rsm:%p %u:%u:%u",
13101 				    tp, bbr, len, sb_offset, sbavail(sb), rsm,
13117 			rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
13158 					panic("tp:%p bbr:%p len:%u moff:%u sbavail:%u rsm:%p snd_una:%u rsm_start:%u flg:%x %u:%u:%u sr:%d ",
13159 					    tp, bbr, len, moff,
13166 					panic("tp:%p bbr:%p len:%u moff:%u sbavail:%u sb_offset:%u snd_una:%u",
13167 					    tp, bbr, len, moff, sbavail(sb), sb_offset, tp->snd_una);
13239 			bbr_log_enobuf_jmp(bbr, len, cts, __LINE__, len, 0, 0);
13492 		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
13494 		log.u_bbr.flex1 = bbr->r_ctl.rc_hptsi_agg_delay;
13495 		log.u_bbr.flex2 = (bbr->r_recovery_bw << 3);
13498 		log.u_bbr.flex5 = bbr->rc_past_init_win;
13500 		log.u_bbr.flex5 |= bbr->rc_no_pacing;
13503 		log.u_bbr.flex6 = bbr->r_ctl.rc_pace_max_segs;
13504 		log.u_bbr.flex7 = (bbr->rc_bbr_state << 8) | bbr_state_val(bbr);
13506 		log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg;
13623 		bbr->output_error_seen = 0;
13624 		bbr->oerror_cnt = 0;
13625 		bbr->bbr_segs_rcvd = 0;
13638 				bbr->r_ctl.rc_del_time = cts;
13648 				idx = len / (bbr_hptsi_bytes_min - bbr->rc_last_options);
13662 		bbr_do_send_accounting(tp, bbr, rsm, len, error);
13665 				bbr->r_ctl.rc_tlp_rxt_last_time = cts;
13668 	bbr_log_output(bbr, tp, &to, len, bbr_seq, (uint8_t) flags, error,
13677 	if (bbr->rc_in_persist == 0) {
13691 			bbr_log_progress_event(bbr, tp, ticks, PROGRESS_START, __LINE__);
13692 			bbr->rc_tp->t_acktime  = ticks;
13730 			bbr_log_progress_event(bbr, tp, ticks, PROGRESS_START, __LINE__);
13731 			bbr->rc_tp->t_acktime = ticks;
13749 		bbr->r_ctl.rc_hptsi_agg_delay = 0;
13750 		bbr->r_ctl.rc_agg_early = 0;
13751 		bbr->r_agg_early_set = 0;
13752 		bbr->output_error_seen = 1;
13753 		if (bbr->oerror_cnt < 0xf)
13754 			bbr->oerror_cnt++;
13755 		if (bbr_max_net_error_cnt && (bbr->oerror_cnt >= bbr_max_net_error_cnt)) {
13766 			if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {
13767 				tp->snd_cwnd = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
13768 								    bbr->r_ctl.rc_lost_bytes)) - maxseg;
13772 			slot = (bbr_error_base_paceout + 1) << bbr->oerror_cnt;
13774 			if (bbr->bbr_hdrw_pacing)
13801 				bbr_log_msgsize_fail(bbr, tp, len, maxseg, mtu, csum_flags, tso, cts);
13817 					bbr_log_msgsize_fail(bbr, tp, len, maxseg, mtu, 0, tso, cts);
13824 					slot = bbr_get_pacing_delay(bbr,
13825 					    bbr->r_ctl.rc_bbr_hptsi_gain,
13828 						slot = (bbr_error_base_paceout + 2) << bbr->oerror_cnt;
13830 					slot = (bbr_error_base_paceout + 2) << bbr->oerror_cnt;
13831 				bbr->rc_output_starts_timer = 1;
13832 				bbr_start_hpts_timer(bbr, tp, cts, 10, slot,
13850 			slot = (bbr_error_base_paceout + 3) << bbr->oerror_cnt;
13851 			bbr->rc_output_starts_timer = 1;
13852 			bbr_start_hpts_timer(bbr, tp, cts, 11, slot, 0);
13859 	    (bbr->rc_in_persist == 0)) {
13868 	if ((bbr->bbr_hdw_pace_ena) &&
13869 	    (bbr->bbr_attempt_hdwr_pace == 0) &&
13870 	    (bbr->rc_past_init_win) &&
13871 	    (bbr->rc_bbr_state != BBR_STATE_STARTUP) &&
13872 	    (get_filter_value(&bbr->r_ctl.rc_delrate)) &&
13885 		rate_wanted = bbr_get_hardware_rate(bbr);
13886 		bbr->bbr_attempt_hdwr_pace = 1;
13887 		bbr->r_ctl.crte = tcp_set_pacing_rate(bbr->rc_tp,
13892 		if (bbr->r_ctl.crte) {
13893 			bbr_type_log_hdwr_pacing(bbr,
13894 						 bbr->r_ctl.crte->ptbl->rs_ifp,
13896 						 bbr->r_ctl.crte->rate,
13901 			bbr->bbr_hdrw_pacing = 1;
13903 			if (bbr->r_ctl.crte->rate < rate_wanted) {
13905 				bbr_setup_less_of_rate(bbr, cts,
13906 						       bbr->r_ctl.crte->rate, rate_wanted);
13909 				bbr->gain_is_limited = 0;
13910 				bbr->skip_gain = 0;
13912 			tcp_bbr_tso_size_check(bbr, cts);
13914 			bbr_type_log_hdwr_pacing(bbr,
13922 	if (bbr->bbr_hdrw_pacing) {
13931 			bbr->bbr_hdrw_pacing = 0;
13940 			bbr->bbr_hdrw_pacing = 0;
13941 			bbr->bbr_attempt_hdwr_pace = 0;
13942 			tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp);
13943 			tcp_bbr_tso_size_check(bbr, cts);
13956 	    (bbr->r_ctl.rc_pace_max_segs > tp->t_maxseg) &&
13963 	    (bbr->rc_in_persist == 0) &&
13964 	    (tot_len < bbr->r_ctl.rc_pace_max_segs)) {
13993 		slot = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0);
13994 		if (bbr->rc_no_pacing)
13999 	if (bbr->rc_use_google == 0)
14000 		bbr_check_bbr_for_state(bbr, cts, __LINE__, 0);
14001 	bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
14002 							bbr->r_ctl.rc_lost_bytes)));
14003 	bbr->rc_output_starts_timer = 1;
14004 	if (bbr->bbr_use_rack_cheat &&
14006 	     ((bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts)) != NULL))) {
14011 	if (bbr->bbr_hdrw_pacing && (bbr->hw_pacing_set == 0)) {
14017 		bbr->r_ctl.bbr_hdwr_cnt_noset_snt++;
14018 		if (bbr->r_ctl.bbr_hdwr_cnt_noset_snt >= bbr_hdwr_pacing_delay_cnt) {
14019 			bbr->hw_pacing_set = 1;
14020 			tcp_bbr_tso_size_check(bbr, cts);
14023 	bbr_start_hpts_timer(bbr, tp, cts, 12, slot, tot_len);
14052 	struct tcp_bbr *bbr;
14061 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
14062 	maxseg = tp->t_maxseg - bbr->rc_last_options;
14063 	sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
14064 	TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
14075 			    bbr_is_lost(bbr, rsm, bbr->r_ctl.rc_rcvtime)) {
14076 				bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;
14077 				bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;
14085 		bbr->r_ctl.rc_resend = frsm;
14105 	 * change bbr to go to USECs).
14114 	struct tcp_bbr *bbr;
14123 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
14125 	if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
14126 		if (TSTMP_GT(bbr->rc_pacer_started, cts)) {
14127 			toval = bbr->rc_pacer_started - cts;
14132 	} else if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
14133 		if (TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) {
14134 			toval = bbr->r_ctl.rc_timer_exp - cts;
14143 	bbr_log_hpts_diag(bbr, cts, &diag);
14175 	struct tcp_bbr *bbr;
14241 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
14245 		bbr->r_ctl.bbr_hptsi_per_second = optval;
14249 		bbr->r_ctl.bbr_hptsi_segments_delay_tar = optval;
14253 		bbr->r_ctl.bbr_hptsi_segments_max = optval;
14257 		bbr->r_ctl.bbr_hptsi_bytes_min = optval;
14261 		bbr->r_ctl.bbr_cross_over = optval;
14265 		if (optval && (bbr->rc_use_google == 0)) {
14267 			bbr_google_mode_on(bbr);
14273 				bbr->r_ctl.bbr_google_discount = optval;
14275 		} else if ((optval == 0) && (bbr->rc_use_google == 1)) {
14277 			bbr_google_mode_off(bbr);
14283 			bbr->rc_use_ts_limit = 1;
14285 			bbr->rc_use_ts_limit = 0;
14295 			bbr->rc_init_win = optval;
14296 			twin = bbr_initial_cwnd(bbr, tp);
14297 			if ((bbr->rc_past_init_win == 0) && (twin > tp->snd_cwnd))
14307 			bbr->r_ctl.rc_startup_pg = optval;
14308 			if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
14309 				bbr->r_ctl.rc_bbr_hptsi_gain = optval;
14317 			bbr->r_ctl.rc_drain_pg = optval;
14324 			reset_time_small(&bbr->r_ctl.rc_rttprop, (optval * USECS_IN_SECOND));
14331 			bbr->r_ctl.bbr_rttprobe_gain_val = optval;
14338 			bbr->r_ctl.rc_probertt_int = optval;
14345 			bbr->no_pacing_until = 0;
14346 			bbr->rc_no_pacing = 0;
14348 			bbr->no_pacing_until = optval;
14349 			if ((bbr->r_ctl.rc_pkt_epoch < bbr->no_pacing_until) &&
14350 			    (bbr->rc_bbr_state == BBR_STATE_STARTUP)){
14352 				bbr->rc_no_pacing = 1;
14359 		bbr->rc_loss_exit = optval;
14366 		bbr->r_ctl.rc_min_rto_ms = optval;
14370 		bbr->rc_max_rto_sec = optval;
14375 		bbr->r_ctl.rc_min_to = optval;
14381 			bbr->r_ctl.rc_reorder_shift = optval;
14388 		bbr->r_ctl.rc_reorder_fade = optval;
14394 			bbr->rc_tlp_threshold = optval;
14400 		if (bbr->rc_use_google) {
14406 			bbr->bbr_use_rack_cheat = 1;
14408 			bbr->bbr_use_rack_cheat = 0;
14413 			bbr->r_ctl.bbr_hptsi_segments_floor = optval;
14420 			bbr->r_ctl.bbr_utter_max = optval;
14428 			bbr->rc_use_idle_restart = 1;
14430 			bbr->rc_use_idle_restart = 0;
14435 			bbr->bbr_init_win_cheat = 1;
14436 			if (bbr->rc_past_init_win == 0) {
14438 				cts = tcp_get_usecs(&bbr->rc_tv);
14439 				tcp_bbr_tso_size_check(bbr, cts);
14442 			bbr->bbr_init_win_cheat = 0;
14447 			bbr->bbr_hdw_pace_ena = 1;
14448 			bbr->bbr_attempt_hdwr_pace = 0;
14450 			bbr->bbr_hdw_pace_ena = 0;
14452 			if (bbr->r_ctl.crte != NULL) {
14453 				tcp_rel_pacing_rate(bbr->r_ctl.crte, tp);
14454 				bbr->r_ctl.crte = NULL;
14482 		bbr->r_ctl.rc_pkt_delay = optval;
14488 			bbr->rc_resends_use_tso = 1;
14490 			bbr->rc_resends_use_tso = 0;
14495 			bbr->rc_allow_data_af_clo = 1;
14497 			bbr->rc_allow_data_af_clo = 0;
14501 		if (bbr->rc_use_google == 0)
14504 			bbr->r_use_policer = 1;
14506 			bbr->r_use_policer = 0;
14512 			bbr->ts_can_raise = 1;
14514 			bbr->ts_can_raise = 0;
14518 		if (bbr->rc_use_google) {
14522 				bbr->r_ctl.rc_incr_tmrs = 1;
14524 				bbr->r_ctl.rc_incr_tmrs = 0;
14529 		if (bbr->rc_use_google) {
14539 				bbr->r_ctl.rc_inc_tcp_oh = 1;
14541 				bbr->r_ctl.rc_inc_tcp_oh = 0;
14543 				bbr->r_ctl.rc_inc_ip_oh = 1;
14545 				bbr->r_ctl.rc_inc_ip_oh = 0;
14547 				bbr->r_ctl.rc_inc_enet_oh = 1;
14549 				bbr->r_ctl.rc_inc_enet_oh = 0;
14568 	struct tcp_bbr *bbr;
14572 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;
14573 	if (bbr == NULL) {
14585 		optval = bbr->r_ctl.bbr_hptsi_per_second;
14588 		optval = bbr->r_ctl.bbr_hptsi_segments_delay_tar;
14591 		optval = bbr->r_ctl.bbr_hptsi_segments_max;
14594 		optval = bbr->no_pacing_until;
14597 		optval = bbr->r_ctl.bbr_hptsi_bytes_min;
14600 		optval = bbr->r_ctl.bbr_cross_over;
14603 		optval = bbr->rc_use_google;
14606 		optval = bbr->rc_use_ts_limit;
14609 		optval = bbr->rc_init_win;
14612 		optval = bbr->r_ctl.rc_startup_pg;
14615 		optval = bbr->r_ctl.rc_drain_pg;
14618 		optval = bbr->r_ctl.rc_probertt_int;
14621 		optval = (bbr->r_ctl.rc_rttprop.cur_time_limit / USECS_IN_SECOND);
14624 		optval = bbr->r_ctl.bbr_rttprobe_gain_val;
14627 		optval = bbr->rc_loss_exit;
14630 		loptval = get_filter_value(&bbr->r_ctl.rc_delrate);
14633 		optval = bbr->r_ctl.rc_min_rto_ms;
14636 		optval = bbr->rc_max_rto_sec;
14640 		optval = bbr->r_ctl.rc_pace_max_segs;
14644 		optval = bbr->r_ctl.rc_min_to;
14648 		optval = bbr->r_ctl.rc_reorder_shift;
14652 		optval = bbr->r_ctl.rc_reorder_fade;
14656 		optval = bbr->bbr_use_rack_cheat;
14659 		optval = bbr->r_ctl.bbr_hptsi_segments_floor;
14662 		optval = bbr->r_ctl.bbr_utter_max;
14666 		optval = bbr->bbr_init_win_cheat;
14669 		optval = bbr->rc_use_idle_restart;
14673 		optval = bbr->rc_tlp_threshold;
14677 		optval = bbr->r_ctl.rc_pkt_delay;
14680 		optval = bbr->rc_resends_use_tso;
14683 		optval = bbr->rc_allow_data_af_clo;
14689 		optval = bbr->bbr_hdw_pace_ena;
14692 		optval = bbr->r_use_policer;
14695 		optval = bbr->ts_can_raise;
14698 		optval = bbr->r_ctl.rc_incr_tmrs;
14702 		if (bbr->r_ctl.rc_inc_tcp_oh)
14704 		if (bbr->r_ctl.rc_inc_ip_oh)
14706 		if (bbr->r_ctl.rc_inc_enet_oh)