Lines Matching +full:brr +full:- +full:mode
1 /*-
2 * Copyright (c) 2016-2020 Netflix, Inc.
29 * BBR - Congestion Based Congestion Control
159 /* Should the following be dynamic too -- loss wise */
193 * num-measures > min(0) and
227 /* thresholds for reduction on drain in sub-states/drain */
265 * means in non-recovery/retransmission scenarios
266 * cwnd will never be reached by the flight-size.
271 static int32_t bbr_sack_not_required = 0; /* set to one to allow non-sack to use bbr */
292 * - Yuchung Cheng's RACK TCP (for which its named) that
295 * - Reorder Detection of RFC4737 and the Tail-Loss probe draft
297 * - Van Jacobson's et.al BBR.
319 * TCP output is also over-written with a new version since it
331 static int32_t bbr_reorder_fade = 60000000; /* 0 - never fade, def
332 * 60,000,000 - 60 seconds */
385 /* Do we use (nf mode) pkt-epoch to drive us or rttProp? */
388 /* What is the max the 0 - bbr_cross_over MBPS TSO target
397 /* What is the min the 0 - bbr_cross-over MBPS TSO target can be */
401 /* Cross over point from algo-a to algo-b */
488 bbr_log_timer_var(struct tcp_bbr *bbr, int mode, uint32_t cts,
517 return(bbr->rc_bbr_substate);
525 mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options),
526 bbr->r_ctl.rc_pace_max_segs);
539 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_PERSIT;
540 if (tp->t_srtt == 0) {
544 srtt = ((uint64_t)TICKS_2_USEC(tp->t_srtt) >> TCP_RTT_SHIFT);
545 var = ((uint64_t)TICKS_2_USEC(tp->t_rttvar) >> TCP_RTT_SHIFT);
547 TCPT_RANGESET_NOSLOP(ret_val, ((srtt + var) * tcp_backoff[tp->t_rxtshift]),
566 if (bbr->rc_all_timers_stopped) {
570 if (bbr->rc_in_persist) {
574 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
576 ((tp->t_flags & TF_SACK_PERMIT) == 0) ||
577 (tp->t_state < TCPS_ESTABLISHED)) {
580 if (SEQ_LT(tp->snd_una, tp->snd_max) ||
581 sbavail(&tptosocket(tp)->so_snd)) {
585 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
587 idx = rsm->r_rtr_cnt - 1;
588 if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time))
589 tstmp_touse = rsm->r_tim_lastsent[idx];
591 tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time;
593 time_since_sent = cts - tstmp_touse;
595 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RXT;
596 if (tp->t_srtt == 0)
599 tov = ((uint64_t)(TICKS_2_USEC(tp->t_srtt) +
600 ((uint64_t)TICKS_2_USEC(tp->t_rttvar) * (uint64_t)4)) >> TCP_RTT_SHIFT);
601 if (tp->t_rxtshift)
602 tov *= tcp_backoff[tp->t_rxtshift];
604 tov -= time_since_sent;
606 tov = bbr->r_ctl.rc_min_to;
608 (bbr->r_ctl.rc_min_rto_ms * MS_IN_USEC),
609 (bbr->rc_max_rto_sec * USECS_IN_SECOND));
615 if (rsm->r_flags & BBR_ACKED) {
623 if (rsm->r_flags & BBR_SACK_PASSED) {
624 if ((tp->t_flags & TF_SENTFIN) &&
625 ((tp->snd_max - tp->snd_una) == 1) &&
626 (rsm->r_flags & BBR_HAS_FIN)) {
635 idx = rsm->r_rtr_cnt - 1;
636 exp = rsm->r_tim_lastsent[idx] + thresh;
638 to = exp - cts;
639 if (to < bbr->r_ctl.rc_min_to) {
640 to = bbr->r_ctl.rc_min_to;
643 to = bbr->r_ctl.rc_min_to;
647 if (bbr->rc_tlp_in_progress != 0) {
653 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext);
658 if (rsm->r_flags & BBR_HAS_FIN) {
664 idx = rsm->r_rtr_cnt - 1;
665 if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time))
666 tstmp_touse = rsm->r_tim_lastsent[idx];
668 tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time;
670 time_since_sent = cts - tstmp_touse;
675 to = thresh - time_since_sent;
677 to = bbr->r_ctl.rc_min_to;
678 if (to > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {
685 if ((bbr->rc_tlp_rtx_out == 1) &&
686 (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq)) {
691 bbr->rc_tlp_rtx_out = 0;
694 if (rsm->r_start != bbr->r_ctl.rc_last_tlp_seq) {
699 bbr->r_ctl.rc_tlp_seg_send_cnt = 0;
700 bbr->r_ctl.rc_last_tlp_seq = rsm->r_start;
705 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RACK;
708 if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) {
716 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_TLP;
725 return (bbr->r_ctl.rc_pace_min_segs - bbr->rc_last_options);
744 if ((tp->t_state == TCPS_CLOSED) ||
745 (tp->t_state == TCPS_LISTEN)) {
748 stopped = bbr->rc_tmr_stopped;
749 if (stopped && TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) {
750 left = bbr->r_ctl.rc_timer_exp - cts;
752 bbr->r_ctl.rc_hpts_flags = 0;
753 bbr->r_ctl.rc_timer_exp = 0;
754 prev_delay = bbr->r_ctl.rc_last_delay_val;
755 if (bbr->r_ctl.rc_last_delay_val &&
762 slot = bbr->r_ctl.rc_last_delay_val;
763 if (TSTMP_GT(cts, bbr->rc_pacer_started)) {
765 delay_calc = cts - bbr->rc_pacer_started;
767 slot -= delay_calc;
771 if (bbr->r_agg_early_set) {
772 bbr_log_pacing_delay_calc(bbr, 0, bbr->r_ctl.rc_agg_early, cts, slot, 0, bbr->r_agg_early_set, 2);
773 slot += bbr->r_ctl.rc_agg_early;
774 bbr->r_ctl.rc_agg_early = 0;
775 bbr->r_agg_early_set = 0;
778 if (bbr->r_ctl.rc_hptsi_agg_delay) {
779 if (slot > bbr->r_ctl.rc_hptsi_agg_delay) {
781 slot -= bbr->r_ctl.rc_hptsi_agg_delay;
782 bbr->r_ctl.rc_hptsi_agg_delay = 0;
785 bbr->r_ctl.rc_hptsi_agg_delay -= slot;
786 bbr->r_ctl.rc_last_delay_val = slot = 100;
789 bbr->r_ctl.rc_last_delay_val = slot;
791 if (tp->t_flags & TF_DELACK) {
792 if (bbr->rc_in_persist == 0) {
812 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;
818 bbr->r_ctl.rc_hpts_flags |= PACE_PKT_OUTPUT;
822 * wheel, we resort to a keep-alive timer if its configured.
826 if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) &&
827 (tp->t_state <= TCPS_CLOSING)) {
830 * del-ack), we don't have segments being paced. So
833 if (TCPS_HAVEESTABLISHED(tp->t_state)) {
838 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_KEEP;
842 (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK)) {
848 * keep-alive, delayed_ack we keep track of what was left
854 if (bbr->r_ctl.rc_incr_tmrs && slot &&
855 (bbr->r_ctl.rc_hpts_flags & (PACE_TMR_TLP|PACE_TMR_RXT))) {
876 * Hack alert for now we can't time-out over 2147 seconds (a
881 bbr->r_ctl.rc_timer_exp = cts + hpts_timeout;
883 bbr->r_ctl.rc_timer_exp = 0;
885 (bbr->rc_use_google ||
886 bbr->output_error_seen ||
892 bbr->rc_tp->t_flags2 |= TF2_MBUF_QUEUE_READY;
893 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) &&
894 (bbr->rc_cwnd_limited == 0)) {
900 tp->t_flags2 |= TF2_DONT_SACK_QUEUE;
902 tp->t_flags2 &= ~TF2_DONT_SACK_QUEUE;
903 bbr->rc_pacer_started = cts;
907 bbr->rc_timer_first = 0;
908 bbr->bbr_timer_src = frm;
921 * on a keep-alive timer and a request comes in for
925 bbr->rc_pacer_started = cts;
926 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) &&
927 (bbr->rc_cwnd_limited == 0)) {
933 tp->t_flags2 |= (TF2_MBUF_QUEUE_READY |
937 tp->t_flags2 &= ~(TF2_MBUF_QUEUE_READY |
940 bbr->bbr_timer_src = frm;
943 bbr->rc_timer_first = 1;
945 bbr->rc_tmr_stopped = 0;
962 tmr_up = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK;
963 if (bbr->rc_in_persist && (tmr_up == PACE_TMR_PERSIT))
965 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
966 if (((rsm == NULL) || (tp->t_state < TCPS_ESTABLISHED)) &&
971 inp = bbr->rc_inp;
974 if (tp->t_flags & TF_DELACK) {
982 inp->inp_socket->so_options & SO_KEEPALIVE) &&
983 (tp->t_state <= TCPS_CLOSING)) &&
985 (tp->snd_max == tp->snd_una)) {
990 if (rsm && (rsm->r_flags & BBR_SACK_PASSED)) {
991 if ((tp->t_flags & TF_SENTFIN) &&
992 ((tp->snd_max - tp->snd_una) == 1) &&
993 (rsm->r_flags & BBR_HAS_FIN)) {
1006 } else if (SEQ_GT(tp->snd_max, tp->snd_una) &&
1010 * Either a TLP or RXT is fine if no sack-passed is in place
1023 if (SEQ_GT(tp->snd_max, tp->snd_una) &&
1042 if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) {
1046 bbr_start_hpts_timer(bbr, tp, cts, 1, bbr->r_ctl.rc_last_delay_val,
1051 * timer. We don't bother with keep-alive, since when we
1052 * jump through the output, it will start the keep-alive if
1055 * We only need a delayed-ack added and or the hpts_timeout.
1058 if (tp->t_flags & TF_DELACK) {
1061 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;
1065 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;
1071 bbr->r_ctl.rc_timer_exp = cts + hpts_timeout;
1093 * time-stamp taken in theory earlier return the difference. The
1098 return (cts - earlier_time);
1111 return (cts - earlier_time);
1121 if (error || req->newptr == NULL)
1146 printf("Clearing BBR out-size counters\n");
1182 "How many mss's are outstanding during probe-rtt");
1187 "If RTT has not shrank in this many micro-seconds enter probe-rtt");
1192 "How many microseconds in probe-rtt");
1207 "If we keep setting new low rtt's but delay going in probe-rtt can we force in??");
1212 "In NF mode, do we imitate google_mode and set the rttProp on entry to probe-rtt?");
1217 "Can we dynamically adjust the probe-rtt limits and times?");
1305 "For 0 -> 24Mbps what is floor number of segments for TSO");
1310 "For 0 -> 24Mbps what is top number of segments for TSO");
1392 "How many pkt-epoch's (0 is off) do we need before pacing is on?");
1402 "Should google mode not use retransmission measurements for the b/w estimation?");
1424 "Do we use a pkt-epoch for substate if 0 rttProp?");
1429 "What increase in RTT triggers us to stop ignoring no-loss and possibly exit startup?");
1469 "How many packet-epochs does the b/w delivery rate last?");
1474 "Does our sub-state drain invoke app limited if its long?");
1479 "Should we set/recover cwnd for sub-state drain?");
1484 "Should we set/recover cwnd for main-state drain?");
1489 "Should we allow google probe-bw/drain to exit early at flight target?");
1494 "Should we have losses exit gain of probebw in google mode??");
1558 "What is the high-speed min cwnd (rttProp under 1ms)");
1651 "Maximum RTO in seconds -- should be at least as large as min_rto");
1708 "Do we call the policer detection code from a rack-timeout?");
1713 "What packet epoch do we do false-positive detection at (0=no)?");
1788 "Total number of enobufs for non-hardware paced flows");
1852 l->cur_del_rate = bbr->r_ctl.rc_bbr_cur_del_rate;
1853 l->delRate = get_filter_value(&bbr->r_ctl.rc_delrate);
1854 l->rttProp = get_filter_value_small(&bbr->r_ctl.rc_rttprop);
1855 l->bw_inuse = bbr_get_bw(bbr);
1856 l->inflight = ctf_flight_size(bbr->rc_tp,
1857 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
1858 l->applimited = bbr->r_ctl.r_app_limited_until;
1859 l->delivered = bbr->r_ctl.rc_delivered;
1860 l->timeStamp = cts;
1861 l->lost = bbr->r_ctl.rc_lost;
1862 l->bbr_state = bbr->rc_bbr_state;
1863 l->bbr_substate = bbr_state_val(bbr);
1864 l->epoch = bbr->r_ctl.rc_rtt_epoch;
1865 l->lt_epoch = bbr->r_ctl.rc_lt_epoch;
1866 l->pacing_gain = bbr->r_ctl.rc_bbr_hptsi_gain;
1867 l->cwnd_gain = bbr->r_ctl.rc_bbr_cwnd_gain;
1868 l->inhpts = tcp_in_hpts(bbr->rc_tp);
1869 l->use_lt_bw = bbr->rc_lt_use_bw;
1870 l->pkts_out = bbr->r_ctl.rc_flight_at_input;
1871 l->pkt_epoch = bbr->r_ctl.rc_pkt_epoch;
1877 if (tcp_bblogging_on(bbr->rc_tp)) {
1880 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
1885 log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_loss_rate;
1887 log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_enters_probertt;
1889 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1890 &bbr->rc_inp->inp_socket->so_rcv,
1891 &bbr->rc_inp->inp_socket->so_snd,
1893 0, &log, false, &bbr->rc_tv);
1898 bbr_log_type_rwnd_collapse(struct tcp_bbr *bbr, int seq, int mode, uint32_t count)
1900 if (tcp_bblogging_on(bbr->rc_tp)) {
1903 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
1906 log.u_bbr.flex8 = mode;
1907 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1908 &bbr->rc_inp->inp_socket->so_rcv,
1909 &bbr->rc_inp->inp_socket->so_snd,
1911 0, &log, false, &bbr->rc_tv);
1919 if (tcp_bblogging_on(bbr->rc_tp)) {
1924 log.u_bbr.flex2 = bbr->r_ctl.rc_hpts_flags;
1925 log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp;
1927 log.u_bbr.flex5 = bbr->rc_in_persist;
1928 log.u_bbr.flex6 = bbr->r_ctl.rc_last_delay_val;
1930 log.u_bbr.flex8 = bbr->rc_in_persist;
1933 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1934 &bbr->rc_inp->inp_socket->so_rcv,
1935 &bbr->rc_inp->inp_socket->so_snd,
1937 tlen, &log, false, &bbr->rc_tv);
1944 if (tcp_bblogging_on(bbr->rc_tp)) {
1947 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
1949 log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent;
1950 log.u_bbr.flex3 = bbr->r_ctl.rc_recovery_start;
1951 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1952 &bbr->rc_inp->inp_socket->so_rcv,
1953 &bbr->rc_inp->inp_socket->so_snd,
1955 0, &log, false, &bbr->rc_tv);
1971 &bbr->rc_inp->inp_socket->so_rcv,
1972 &bbr->rc_inp->inp_socket->so_snd,
1974 0, &log, false, &bbr->rc_tv);
1981 if (tcp_bblogging_on(bbr->rc_tp)) {
1986 if (bbr->rc_inp->inp_socket) {
1987 r = &bbr->rc_inp->inp_socket->so_rcv;
1988 s = &bbr->rc_inp->inp_socket->so_snd;
1993 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2004 if (tcp_bblogging_on(bbr->rc_tp)) {
2010 log.u_bbr.flex3 = bbr->r_ctl.rc_bbr_lastbtlbw;
2011 log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_rtt;
2012 log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch;
2013 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;
2016 log.u_bbr.inflight = bbr->r_ctl.r_measurement_count;
2017 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2018 &bbr->rc_inp->inp_socket->so_rcv,
2019 &bbr->rc_inp->inp_socket->so_snd,
2021 0, &log, false, &bbr->rc_tv);
2028 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2032 log.u_bbr.flex1 = bbr->r_ctl.rc_lost;
2033 log.u_bbr.flex2 = bbr->rc_inp->inp_socket->so_snd.sb_lowat;
2034 log.u_bbr.flex3 = bbr->rc_inp->inp_socket->so_snd.sb_hiwat;
2036 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2037 &bbr->rc_inp->inp_socket->so_rcv,
2038 &bbr->rc_inp->inp_socket->so_snd,
2040 0, &log, false, &bbr->rc_tv);
2047 if (tcp_bblogging_on(bbr->rc_tp)) {
2050 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2051 log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state;
2054 log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs;
2056 log.u_bbr.flex6 = bbr->r_ctl.rc_pace_min_segs;
2057 log.u_bbr.flex7 = bbr->rc_last_options;
2059 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2060 &bbr->rc_inp->inp_socket->so_rcv,
2061 &bbr->rc_inp->inp_socket->so_snd,
2063 0, &log, false, &bbr->rc_tv);
2071 if (tcp_bblogging_on(bbr->rc_tp)) {
2076 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;
2077 log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int;
2082 log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch;
2083 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;
2084 log.u_bbr.flex7 = (bbr->r_ctl.rc_target_at_state/1000);
2085 log.u_bbr.lt_epoch = bbr->r_ctl.rc_level_state_extra;
2086 log.u_bbr.pkts_out = bbr->r_ctl.rc_target_at_state;
2087 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2088 &bbr->rc_inp->inp_socket->so_rcv,
2089 &bbr->rc_inp->inp_socket->so_snd,
2091 0, &log, false, &bbr->rc_tv);
2099 if (tcp_bblogging_on(bbr->rc_tp)) {
2104 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;
2105 log.u_bbr.flex3 = bbr->r_ctl.last_in_probertt;
2108 log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state;
2111 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2112 &bbr->rc_inp->inp_socket->so_rcv,
2113 &bbr->rc_inp->inp_socket->so_snd,
2115 0, &log, false, &bbr->rc_tv);
2122 if (tcp_bblogging_on(bbr->rc_tp)) {
2125 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2126 log.u_bbr.flex1 = bbr->r_ctl.rc_recovery_start;
2127 log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent;
2128 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2129 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2130 &bbr->rc_inp->inp_socket->so_rcv,
2131 &bbr->rc_inp->inp_socket->so_snd,
2133 0, &log, false, &bbr->rc_tv);
2141 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2144 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2152 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2153 &bbr->rc_inp->inp_socket->so_rcv,
2154 &bbr->rc_inp->inp_socket->so_snd,
2156 0, &log, false, &bbr->rc_tv);
2167 if (tcp_bblogging_on(bbr->rc_tp)) {
2170 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2172 log.u_bbr.flex2 = bbr->r_ctl.rc_bbr_state_time;
2173 log.u_bbr.flex3 = bbr->r_ctl.rc_ack_hdwr_delay;
2174 log.u_bbr.flex4 = bbr->rc_tp->ts_offset;
2175 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2176 log.u_bbr.pkts_out = tcp_tv_to_mssectick(&bbr->rc_tv);
2179 log.u_bbr.flex8 = bbr->rc_ack_was_delayed;
2180 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2181 &bbr->rc_inp->inp_socket->so_rcv,
2182 &bbr->rc_inp->inp_socket->so_snd,
2184 0, &log, false, &bbr->rc_tv);
2191 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2198 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2199 &bbr->rc_inp->inp_socket->so_rcv,
2200 &bbr->rc_inp->inp_socket->so_snd,
2202 0, &log, false, &bbr->rc_tv);
2208 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2212 log.u_bbr.flex1 = bbr->rc_tp->ts_recent_age;
2213 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;
2214 log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int;
2215 log.u_bbr.flex4 = bbr->r_ctl.rc_went_idle_time;
2216 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2217 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2218 &bbr->rc_inp->inp_socket->so_rcv,
2219 &bbr->rc_inp->inp_socket->so_snd,
2221 0, &log, false, &bbr->rc_tv);
2229 if (tcp_bblogging_on(bbr->rc_tp)) {
2235 log.u_bbr.flex2 = bbr->r_ctl.rc_lost_bytes;
2239 log.u_bbr.flex3 = m->m_flags;
2240 if (m->m_flags & M_TSTMP) {
2248 if (m->m_flags & M_TSTMP_LRO) {
2263 log.u_bbr.flex4 = bbr->r_ctl.rc_target_at_state;
2264 log.u_bbr.flex7 = bbr->r_wanted_output;
2265 log.u_bbr.flex8 = bbr->rc_in_persist;
2266 TCP_LOG_EVENTP(bbr->rc_tp, th,
2267 &bbr->rc_inp->inp_socket->so_rcv,
2268 &bbr->rc_inp->inp_socket->so_snd,
2270 tlen, &log, true, &bbr->rc_tv);
2277 if (tcp_bblogging_on(bbr->rc_tp)) {
2283 log.u_bbr.flex3 = bbr->r_ctl.rc_last_delay_val;
2284 log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags;
2285 log.u_bbr.flex5 = bbr->r_ctl.rc_timer_exp;
2286 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_bytes;
2287 log.u_bbr.flex7 = bbr->r_wanted_output;
2288 log.u_bbr.flex8 = bbr->rc_in_persist;
2289 log.u_bbr.pkts_out = bbr->r_ctl.highest_hdwr_delay;
2290 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2291 &bbr->rc_inp->inp_socket->so_rcv,
2292 &bbr->rc_inp->inp_socket->so_snd,
2294 0, &log, true, &bbr->rc_tv);
2302 if (tcp_bblogging_on(bbr->rc_tp)) {
2310 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2311 &bbr->rc_inp->inp_socket->so_rcv,
2312 &bbr->rc_inp->inp_socket->so_snd,
2314 len, &log, true, &bbr->rc_tv);
2321 if (tcp_bblogging_on(bbr->rc_tp)) {
2327 log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp;
2328 log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags;
2330 log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state;
2332 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2333 &bbr->rc_inp->inp_socket->so_rcv,
2334 &bbr->rc_inp->inp_socket->so_snd,
2336 0, &log, false, &bbr->rc_tv);
2343 if (tcp_bblogging_on(bbr->rc_tp)) {
2348 log.u_bbr.flex1 = bbr->bbr_timer_src;
2350 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;
2351 ar = (uintptr_t)(bbr->r_ctl.rc_resend);
2355 ar = (uintptr_t)bbr->r_ctl.rc_resend;
2358 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2360 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2361 &bbr->rc_inp->inp_socket->so_rcv,
2362 &bbr->rc_inp->inp_socket->so_snd,
2364 0, &log, false, &bbr->rc_tv);
2371 if (tcp_bblogging_on(bbr->rc_tp)) {
2379 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2380 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;
2382 log.u_bbr.cur_del_rate = bbr->r_ctl.rc_bbr_lastbtlbw;
2383 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2384 &bbr->rc_inp->inp_socket->so_rcv,
2385 &bbr->rc_inp->inp_socket->so_snd,
2387 0, &log, false, &bbr->rc_tv);
2394 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2398 log.u_bbr.flex1 = diag->p_nxt_slot;
2399 log.u_bbr.flex2 = diag->p_cur_slot;
2400 log.u_bbr.flex3 = diag->slot_req;
2401 log.u_bbr.flex4 = diag->inp_hptsslot;
2402 log.u_bbr.flex5 = diag->slot_remaining;
2403 log.u_bbr.flex6 = diag->need_new_to;
2404 log.u_bbr.flex7 = diag->p_hpts_active;
2405 log.u_bbr.flex8 = diag->p_on_min_sleep;
2407 log.u_bbr.epoch = diag->have_slept;
2408 log.u_bbr.lt_epoch = diag->yet_to_sleep;
2409 log.u_bbr.pkts_out = diag->co_ret;
2410 log.u_bbr.applimited = diag->hpts_sleep_time;
2411 log.u_bbr.delivered = diag->p_prev_slot;
2412 log.u_bbr.inflight = diag->p_runningslot;
2413 log.u_bbr.bw_inuse = diag->wheel_slot;
2414 log.u_bbr.rttProp = diag->wheel_cts;
2415 log.u_bbr.delRate = diag->maxslots;
2416 log.u_bbr.cur_del_rate = diag->p_curtick;
2418 log.u_bbr.cur_del_rate |= diag->p_lasttick;
2419 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2420 &bbr->rc_inp->inp_socket->so_rcv,
2421 &bbr->rc_inp->inp_socket->so_snd,
2423 0, &log, false, &bbr->rc_tv);
2428 bbr_log_timer_var(struct tcp_bbr *bbr, int mode, uint32_t cts, uint32_t time_since_sent, uint32_t srtt,
2431 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2435 log.u_bbr.flex1 = bbr->rc_tp->t_rttvar;
2440 log.u_bbr.flex6 = bbr->rc_tp->t_srtt;
2441 log.u_bbr.flex8 = mode;
2442 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2443 &bbr->rc_inp->inp_socket->so_rcv,
2444 &bbr->rc_inp->inp_socket->so_snd,
2446 0, &log, false, &bbr->rc_tv);
2454 if (tcp_bblogging_on(bbr->rc_tp)) {
2469 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2470 &bbr->rc_inp->inp_socket->so_rcv,
2471 &bbr->rc_inp->inp_socket->so_snd,
2473 len, &log, false, &bbr->rc_tv);
2480 if (tcp_bblogging_on(bbr->rc_tp)) {
2485 log.u_bbr.flex1 = bbr->bbr_timer_src;
2487 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;
2489 log.u_bbr.flex5 = bbr->rc_tp->t_hpts_slot;
2490 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2491 log.u_bbr.pkts_out = bbr->rc_tp->t_flags2;
2493 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2494 &bbr->rc_inp->inp_socket->so_rcv,
2495 &bbr->rc_inp->inp_socket->so_snd,
2497 0, &log, false, &bbr->rc_tv);
2504 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2510 log.u_bbr.flex3 = bbr->r_ctl.rc_reorder_ts;
2511 log.u_bbr.flex4 = rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)];
2512 log.u_bbr.flex5 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2514 log.u_bbr.flex7 = bbr->r_ctl.rc_reorder_shift;
2516 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2517 &bbr->rc_inp->inp_socket->so_rcv,
2518 &bbr->rc_inp->inp_socket->so_snd,
2520 0, &log, false, &bbr->rc_tv);
2527 if (tcp_bblogging_on(bbr->rc_tp)) {
2532 log.u_bbr.flex2 = bbr->bbr_timer_src;
2533 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;
2534 log.u_bbr.flex4 = bbr->rc_in_persist;
2535 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2536 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2538 log.u_bbr.pkts_out = bbr->rc_pacer_started;
2539 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2540 &bbr->rc_inp->inp_socket->so_rcv,
2541 &bbr->rc_inp->inp_socket->so_snd,
2543 0, &log, false, &bbr->rc_tv);
2550 if (tcp_bblogging_on(bbr->rc_tp)) {
2553 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2554 log.u_bbr.flex1 = bbr->r_ctl.bbr_peer_tsratio;
2559 log.u_bbr.flex7 = bbr->rc_ts_clock_set;
2560 log.u_bbr.flex8 = bbr->rc_ts_cant_be_used;
2561 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2562 &bbr->rc_inp->inp_socket->so_rcv,
2563 &bbr->rc_inp->inp_socket->so_snd,
2565 0, &log, false, &bbr->rc_tv);
2572 if (tcp_bblogging_on(bbr->rc_tp)) {
2579 log.u_bbr.flex4 = bbr->r_ctl.bbr_hptsi_bytes_min;
2582 log.u_bbr.flex7 = bbr->rc_no_pacing;
2584 log.u_bbr.flex7 |= bbr->rc_past_init_win;
2586 log.u_bbr.flex8 = 0x80 | bbr->rc_use_google;
2588 log.u_bbr.flex8 = bbr->rc_use_google;
2589 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2590 &bbr->rc_inp->inp_socket->so_rcv,
2591 &bbr->rc_inp->inp_socket->so_snd,
2593 0, &log, false, &bbr->rc_tv);
2601 if (tcp_bblogging_on(bbr->rc_tp)) {
2606 log.u_bbr.flex2 = rsm->r_start;
2607 log.u_bbr.flex3 = rsm->r_end;
2608 log.u_bbr.flex4 = rsm->r_delivered;
2609 log.u_bbr.flex5 = rsm->r_rtr_cnt;
2610 log.u_bbr.flex6 = rsm->r_dupack;
2611 log.u_bbr.flex7 = rsm->r_tim_lastsent[0];
2612 log.u_bbr.flex8 = rsm->r_flags;
2615 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2616 &bbr->rc_inp->inp_socket->so_rcv,
2617 &bbr->rc_inp->inp_socket->so_snd,
2619 0, &log, false, &bbr->rc_tv);
2630 if (tcp_bblogging_on(bbr->rc_tp)) {
2644 if (bbr->rc_ack_was_delayed)
2645 log.u_bbr.epoch = bbr->r_ctl.rc_ack_hdwr_delay;
2648 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2649 &bbr->rc_inp->inp_socket->so_rcv,
2650 &bbr->rc_inp->inp_socket->so_snd,
2652 flex2, &log, false, &bbr->rc_tv);
2661 if (/*bbr_verbose_logging && */tcp_bblogging_on(bbr->rc_tp)) {
2669 log.u_bbr.flex5 = bbr->r_ctl.rc_lt_lost;
2670 log.u_bbr.flex6 = bbr->r_ctl.rc_lt_del;
2671 log.u_bbr.flex7 = bbr->rc_lt_is_sampling;
2673 log.u_bbr.bw_inuse = bbr->r_ctl.rc_lt_bw;
2674 if (bbr->rc_lt_use_bw == 0)
2675 log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch;
2677 log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use;
2678 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2679 &bbr->rc_inp->inp_socket->so_rcv,
2680 &bbr->rc_inp->inp_socket->so_snd,
2682 0, &log, false, &bbr->rc_tv);
2689 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2692 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2695 log.u_bbr.flex3 = tp->t_maxunacktime;
2696 log.u_bbr.flex4 = tp->t_acktime;
2698 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2699 &bbr->rc_inp->inp_socket->so_rcv,
2700 &bbr->rc_inp->inp_socket->so_snd,
2702 0, &log, false, &bbr->rc_tv);
2711 if (tcp_bblogging_on(bbr->rc_tp)) {
2723 log.u_bbr.flex8 = bbr->skip_gain;
2725 log.u_bbr.flex8 |= bbr->gain_is_limited;
2727 log.u_bbr.flex8 |= bbr->bbr_hdrw_pacing;
2728 log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg;
2729 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2730 &bbr->rc_inp->inp_socket->so_rcv,
2731 &bbr->rc_inp->inp_socket->so_snd,
2733 0, &log, false, &bbr->rc_tv);
2740 if (tcp_bblogging_on(bbr->rc_tp)) {
2748 log.u_bbr.flex5 = bbr->r_ctl.rc_last_delay_val;
2749 log.u_bbr.flex6 = bbr->r_ctl.rc_hptsi_agg_delay;
2750 log.u_bbr.flex7 = (0x0000ffff & bbr->r_ctl.rc_hpts_flags);
2751 log.u_bbr.flex8 = bbr->rc_in_persist;
2752 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2753 &bbr->rc_inp->inp_socket->so_rcv,
2754 &bbr->rc_inp->inp_socket->so_snd,
2756 len, &log, false, &bbr->rc_tv);
2763 if (tcp_bblogging_on(bbr->rc_tp)) {
2767 log.u_bbr.flex1 = bbr->r_ctl.rc_delivered;
2769 log.u_bbr.flex3 = bbr->r_ctl.rc_lowest_rtt;
2775 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2776 &bbr->rc_inp->inp_socket->so_rcv,
2777 &bbr->rc_inp->inp_socket->so_snd,
2779 0, &log, false, &bbr->rc_tv);
2786 if (tcp_bblogging_on(bbr->rc_tp)) {
2790 log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state;
2791 log.u_bbr.flex2 = (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
2792 log.u_bbr.flex3 = bbr->r_ctl.gain_epoch;
2793 log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs;
2794 log.u_bbr.flex5 = bbr->r_ctl.rc_pace_min_segs;
2795 log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_state_atflight;
2798 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2799 &bbr->rc_inp->inp_socket->so_rcv,
2800 &bbr->rc_inp->inp_socket->so_snd,
2802 0, &log, false, &bbr->rc_tv);
2809 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2812 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2813 /* R-HU */
2821 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2822 &bbr->rc_inp->inp_socket->so_rcv,
2823 &bbr->rc_inp->inp_socket->so_snd,
2825 0, &log, false, &bbr->rc_tv);
2837 bw = get_filter_value(&bbr->r_ctl.rc_delrate);
2848 if (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_pktepoch)
2849 lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lost_at_pktepoch;
2852 del = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_pkt_epoch_del;
2863 bbr->r_ctl.rc_pkt_epoch_loss_rate = (uint32_t)calclr;
2864 if (IN_RECOVERY(bbr->rc_tp->t_flags))
2865 bbr->r_ctl.recovery_lr += (uint32_t)calclr;
2866 bbr->r_ctl.rc_pkt_epoch++;
2867 if (bbr->rc_no_pacing &&
2868 (bbr->r_ctl.rc_pkt_epoch >= bbr->no_pacing_until)) {
2869 bbr->rc_no_pacing = 0;
2872 bbr->r_ctl.rc_pkt_epoch_rtt = bbr_calc_time(cts, bbr->r_ctl.rc_pkt_epoch_time);
2873 bbr->r_ctl.rc_pkt_epoch_time = cts;
2876 bbr->r_ctl.rc_pkt_epoch_del = bbr->r_ctl.rc_delivered;
2877 bbr->r_ctl.rc_lost_at_pktepoch = bbr->r_ctl.rc_lost;
2886 bbr->r_ctl.rc_rtt_epoch++;
2887 epoch_time = cts - bbr->r_ctl.rc_rcv_epoch_start;
2889 bbr->r_ctl.rc_rcv_epoch_start = cts;
2895 if (SEQ_GEQ(rsm->r_delivered, bbr->r_ctl.rc_pkt_epoch_del)) {
2896 bbr->rc_is_pkt_epoch_now = 1;
2920 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
2922 rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop);
2925 min_bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) *
2928 if (min_bw < bbr->r_ctl.rc_initial_hptsi_bw) {
2929 min_bw = bbr->r_ctl.rc_initial_hptsi_bw;
2932 } else if (bbr->rc_tp->t_srtt != 0) {
2937 min_bw = bbr->r_ctl.rc_initial_hptsi_bw;
2942 if ((bbr->rc_past_init_win == 0) &&
2943 (bbr->r_ctl.rc_delivered > bbr_initial_cwnd(bbr, bbr->rc_tp)))
2944 bbr->rc_past_init_win = 1;
2945 if ((bbr->rc_use_google) && (bbr->r_ctl.r_measurement_count >= 1))
2948 ((bbr->r_ctl.r_measurement_count < bbr_min_measurements_req) ||
2949 (bbr->rc_past_init_win == 0))) {
2953 rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop);
2960 bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) *
2963 if (bw < bbr->r_ctl.rc_initial_hptsi_bw) {
2964 bw = bbr->r_ctl.rc_initial_hptsi_bw;
2968 bw = bbr->r_ctl.rc_initial_hptsi_bw;
2978 if (bbr->rc_lt_use_bw)
2979 bw = bbr->r_ctl.rc_lt_bw;
2980 else if (bbr->r_recovery_bw && (bbr->rc_use_google == 0))
2981 bw = bbr->r_ctl.red_bw;
2983 bw = get_filter_value(&bbr->r_ctl.rc_delrate);
3005 bbr->r_ctl.rc_lt_epoch = bbr->r_ctl.rc_pkt_epoch;
3006 bbr->r_ctl.rc_lt_time = bbr->r_ctl.rc_del_time;
3007 bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered;
3008 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
3014 bbr->rc_lt_is_sampling = 0;
3015 bbr->rc_lt_use_bw = 0;
3016 bbr->r_ctl.rc_lt_bw = 0;
3026 if (bbr->r_ctl.rc_lt_bw) {
3028 if (bbr->r_ctl.rc_lt_bw > bw)
3029 diff = bbr->r_ctl.rc_lt_bw - bw;
3031 diff = bw - bbr->r_ctl.rc_lt_bw;
3033 (diff <= (bbr->r_ctl.rc_lt_bw / bbr_lt_bw_ratio))) {
3037 saved_bw = (uint32_t)bbr->r_ctl.rc_lt_bw;
3038 bbr->r_ctl.rc_lt_bw = (bw + bbr->r_ctl.rc_lt_bw) / 2; /* average of two */
3039 bbr->rc_lt_use_bw = 1;
3040 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
3045 bbr->r_ctl.rc_lt_epoch_use = bbr->r_ctl.rc_pkt_epoch;
3054 bbr->r_ctl.rc_lt_bw = bw;
3066 deduct = bbr->r_ctl.rc_level_state_extra / ran;
3067 bbr->r_ctl.rc_level_state_extra -= deduct;
3081 bbr->r_ctl.rc_exta_time_gd = 0;
3082 bbr->rc_hit_state_1 = 0;
3083 bbr->r_ctl.rc_level_state_extra = 0;
3084 ran = arc4random_uniform((BBR_SUBSTATE_COUNT-1));
3089 * we fully enter the state. Note that the (8 - 1 - ran) assures that
3090 * we return 1 - 7, so we dont return 0 and end up starting in
3093 ret_val = BBR_SUBSTATE_COUNT - 1 - ran;
3095 if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP))
3098 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
3108 if (bbr->r_use_policer == 0)
3110 if (bbr->rc_lt_use_bw) {
3112 diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use;
3117 if (bbr->rc_filled_pipe) {
3119 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
3121 bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
3129 bbr->rc_bbr_state = BBR_STATE_STARTUP;
3131 bbr->r_ctl.rc_bbr_state_time = cts;
3132 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
3133 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;
3134 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;
3138 /* reason 0 is to stop using lt-bw */
3143 /* Not doing false-positive detection */
3149 bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered;
3150 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
3153 lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost;
3154 delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del;
3169 * exhaust its tokens and estimate the steady-state rate allowed by
3171 * over-estimate the bw.
3173 if (bbr->rc_lt_is_sampling == 0) {
3178 bbr->rc_lt_is_sampling = 1;
3183 if (TSTMP_GEQ(bbr->r_ctl.rc_del_time, bbr->r_ctl.rc_lt_time))
3184 d_time = bbr->r_ctl.rc_del_time - bbr->r_ctl.rc_lt_time;
3189 if (bbr->r_ctl.r_app_limited_until) {
3190 /* Can not measure in app-limited state */
3196 diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch;
3203 /* 6 is not_enough time or no-loss */
3223 * tokens are exhausted under-estimates the policed rate.
3226 /* 6 is not_enough time or no-loss */
3231 lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost;
3232 delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del;
3240 /* 6 is not_enough time or no-loss */
3269 bbr->r_ctl.rc_num_maps_alloced++;
3272 if (bbr->r_ctl.rc_free_cnt) {
3274 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);
3275 TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next);
3276 bbr->r_ctl.rc_free_cnt--;
3287 (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {
3289 if (!bbr->alloc_limit_reported) {
3290 bbr->alloc_limit_reported = 1;
3307 bbr->r_ctl.rc_num_split_allocs >= V_tcp_map_split_limit) {
3309 if (!bbr->alloc_limit_reported) {
3310 bbr->alloc_limit_reported = 1;
3320 rsm->r_limit_type = limit_type;
3321 bbr->r_ctl.rc_num_split_allocs++;
3329 if (rsm->r_limit_type) {
3331 bbr->r_ctl.rc_num_split_allocs--;
3333 if (rsm->r_is_smallmap)
3334 bbr->r_ctl.rc_num_small_maps_alloced--;
3335 if (bbr->r_ctl.rc_tlp_send == rsm)
3336 bbr->r_ctl.rc_tlp_send = NULL;
3337 if (bbr->r_ctl.rc_resend == rsm) {
3338 bbr->r_ctl.rc_resend = NULL;
3340 if (bbr->r_ctl.rc_next == rsm)
3341 bbr->r_ctl.rc_next = NULL;
3342 if (bbr->r_ctl.rc_sacklast == rsm)
3343 bbr->r_ctl.rc_sacklast = NULL;
3344 if (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) {
3346 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next);
3347 rsm->r_limit_type = 0;
3348 bbr->r_ctl.rc_free_cnt++;
3351 bbr->r_ctl.rc_num_maps_alloced--;
3384 if (bbr->rc_init_win) {
3385 i_cwnd = bbr->rc_init_win * tp->t_maxseg;
3387 i_cwnd = min((V_tcp_initcwnd_segments * tp->t_maxseg),
3388 max(2 * tp->t_maxseg, 14600));
3390 i_cwnd = min(4 * tp->t_maxseg,
3391 max(2 * tp->t_maxseg, 4380));
3394 if (tp->t_maxseg > 2190)
3395 i_cwnd = 2 * tp->t_maxseg;
3396 else if (tp->t_maxseg > 1095)
3397 i_cwnd = 3 * tp->t_maxseg;
3399 i_cwnd = 4 * tp->t_maxseg;
3414 if ((get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) ||
3417 return (bbr_initial_cwnd(bbr, bbr->rc_tp));
3427 cwnd = (uint32_t)(((bdp * ((uint64_t)gain)) + (uint64_t)(BBR_UNIT - 1)) / ((uint64_t)BBR_UNIT));
3437 mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs);
3442 * fq layer to trap packets in) quanta's per the I-D
3445 cwnd += (bbr_quanta * bbr->r_ctl.rc_pace_max_segs);
3446 if (bbr->rc_use_google) {
3447 if((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&
3452 * is documented no-where except in the code.
3482 if (bbr->r_ctl.rc_inc_tcp_oh) {
3484 seg_oh = (bbr->rc_last_options + sizeof(struct tcphdr));
3486 if (bbr->r_ctl.rc_inc_ip_oh) {
3489 if (bbr->r_is_v6) {
3500 if (bbr->r_ctl.rc_inc_enet_oh) {
3538 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
3539 num_segs = (len + maxseg - 1) / maxseg;
3540 if (bbr->rc_use_google == 0) {
3546 if (bbr->rc_use_google) {
3553 cbw = bw * (uint64_t)(1000 - bbr->r_ctl.bbr_google_discount);
3568 (bbr->rc_use_google == 0) &&
3593 if ((tp->t_flags & TF_GPUTINPROG) &&
3594 SEQ_GEQ(th->th_ack, tp->gput_ack)) {
3604 gput = (int64_t) (th->th_ack - tp->gput_seq) * 8;
3605 time_stamp = max(1, ((bbr->r_ctl.rc_rcvtime - tp->gput_ts) / 1000));
3607 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_GPUT,
3609 if (tp->t_stats_gput_prev > 0)
3610 stats_voi_update_abs_s32(tp->t_stats,
3612 ((gput - tp->t_stats_gput_prev) * 100) /
3613 tp->t_stats_gput_prev);
3614 tp->t_flags &= ~TF_GPUTINPROG;
3615 tp->t_stats_gput_prev = cgput;
3618 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&
3619 ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) {
3620 /* We don't change anything in probe-rtt */
3623 maxseg = tp->t_maxseg - bbr->rc_last_options;
3627 bytes_this_ack -= prev_acked;
3632 if ((bytes_this_ack < maxseg) && bbr->rc_use_google)
3638 cwnd = tp->snd_cwnd;
3639 bw = get_filter_value(&bbr->r_ctl.rc_delrate);
3643 (uint32_t)bbr->r_ctl.rc_bbr_cwnd_gain);
3645 target_cwnd = bbr_initial_cwnd(bbr, bbr->rc_tp);
3646 if (IN_RECOVERY(tp->t_flags) &&
3647 (bbr->bbr_prev_in_rec == 0)) {
3652 bbr->pkt_conservation = 1;
3653 bbr->r_ctl.rc_recovery_start = bbr->r_ctl.rc_rcvtime;
3655 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
3658 if (IN_RECOVERY(tp->t_flags)) {
3661 bbr->bbr_prev_in_rec = 1;
3663 cwnd -= losses;
3669 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
3672 if (bbr->pkt_conservation) {
3675 if (TSTMP_GEQ(bbr->r_ctl.rc_rcvtime, bbr->r_ctl.rc_recovery_start))
3676 time_in = bbr->r_ctl.rc_rcvtime - bbr->r_ctl.rc_recovery_start;
3682 bbr->pkt_conservation = 0;
3688 tp->snd_cwnd = cwnd;
3690 prev_acked, 1, target_cwnd, th->th_ack, line);
3695 bbr->bbr_prev_in_rec = 0;
3696 if ((bbr->rc_use_google == 0) && bbr->r_ctl.restrict_growth) {
3697 bbr->r_ctl.restrict_growth--;
3701 if (bbr->rc_filled_pipe) {
3713 else if (bbr_cwnd_may_shrink || bbr->rc_use_google || bbr->rc_no_pacing)
3721 (bbr->rc_past_init_win == 0)) {
3732 tp->snd_cwnd = max(cwnd, get_min_cwnd(bbr));
3733 bbr_log_type_cwndupd(bbr, saved_bytes, sack_changed, prev_acked, meth, target_cwnd, th->th_ack, line);
3741 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
3744 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <=
3745 tp->snd_cwnd) {
3746 bbr->r_wanted_output = 1;
3757 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
3761 EXIT_RECOVERY(tp->t_flags);
3762 /* Lock in our b/w reduction for the specified number of pkt-epochs */
3763 bbr->r_recovery_bw = 0;
3764 tp->snd_recover = tp->snd_una;
3765 tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);
3766 bbr->pkt_conservation = 0;
3767 if (bbr->rc_use_google == 0) {
3769 * For non-google mode lets
3775 bbr->bbr_prev_in_rec = 0;
3778 if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {
3779 tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent);
3782 /* For probe-rtt case lets fix up its saved_cwnd */
3783 if (bbr->r_ctl.rc_saved_cwnd < bbr->r_ctl.rc_cwnd_on_ent) {
3784 bbr->r_ctl.rc_saved_cwnd = bbr->r_ctl.rc_cwnd_on_ent;
3789 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
3790 if ((bbr->rc_use_google == 0) &&
3804 bbr->r_ctl.recovery_lr, 21,
3806 bbr->r_ctl.rc_red_cwnd_pe,
3810 if (((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&
3813 (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&
3814 bbr->rc_hit_state_1 &&
3816 ((bbr->rc_bbr_state == BBR_STATE_DRAIN) &&
3822 cwnd = bbr->r_ctl.rc_saved_cwnd;
3823 cwnd_p = &bbr->r_ctl.rc_saved_cwnd;
3825 cwnd = tp->snd_cwnd;
3826 cwnd_p = &tp->snd_cwnd;
3828 maxseg = tp->t_maxseg - bbr->rc_last_options;
3830 if (bbr->r_ctl.rc_lost == 0)
3832 else if (bbr->r_ctl.rc_delivered == 0)
3835 lr2use = (uint64_t)bbr->r_ctl.rc_lost * (uint64_t)1000;
3836 lr2use /= bbr->r_ctl.rc_delivered;
3838 lr2use += bbr->r_ctl.recovery_lr;
3845 bbr->r_ctl.restrict_growth += acks_inflight;
3851 newcwnd = roundup((cwnd - val), maxseg);
3867 newcwnd = (get_min_cwnd(bbr) - acks_inflight);
3879 if (tp->snd_cwnd > newcwnd)
3880 tp->snd_cwnd = newcwnd;
3884 bbr->r_ctl.rc_red_cwnd_pe = bbr->r_ctl.rc_pkt_epoch;
3887 bbr->r_ctl.recovery_lr = 0;
3888 if (flight <= tp->snd_cwnd) {
3889 bbr->r_wanted_output = 1;
3891 tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);
3897 bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate);
3899 if (bbr->r_ctl.red_bw > bbr->r_ctl.rc_bbr_cur_del_rate)
3900 bbr->r_ctl.red_bw = bbr->r_ctl.rc_bbr_cur_del_rate;
3901 if (bbr->r_ctl.red_bw < (get_filter_value(&bbr->r_ctl.rc_delrate) / 2))
3902 bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate) / 2;
3913 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_CSIG, type);
3915 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
3918 if (!IN_RECOVERY(tp->t_flags)) {
3919 tp->snd_recover = tp->snd_max;
3921 bbr_set_pktepoch(bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
3922 if (bbr->rc_lt_is_sampling || bbr->rc_lt_use_bw) {
3928 bbr->r_ctl.rc_lt_epoch++;
3930 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
3937 bbr->r_ctl.rc_bbr_last_startup_epoch++;
3939 bbr->r_ctl.rc_cwnd_on_ent = tp->snd_cwnd;
3940 ENTER_RECOVERY(tp->t_flags);
3941 bbr->rc_tlp_rtx_out = 0;
3942 bbr->r_ctl.recovery_lr = bbr->r_ctl.rc_pkt_epoch_loss_rate;
3943 tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);
3944 if (tcp_in_hpts(bbr->rc_tp) &&
3945 ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) == 0)) {
3952 bbr->rc_timer_first = 1;
3953 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
3961 if ((bbr->rc_use_google == 0) &&
3962 (bbr->r_ctl.bbr_rttprobe_gain_val ||
3963 (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT))) {
3964 tp->snd_cwnd = ctf_flight_size(tp,
3965 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
3966 (tp->t_maxseg - bbr->rc_last_options);
3967 if (tp->snd_cwnd < get_min_cwnd(bbr)) {
3969 tp->snd_cwnd = get_min_cwnd(bbr);
3973 bbr_log_type_enter_rec(bbr, rsm->r_start);
3979 bbr_reset_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime);
3980 if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {
3981 tp->snd_cwnd = tp->snd_cwnd_prev;
3982 tp->snd_ssthresh = tp->snd_ssthresh_prev;
3983 tp->snd_recover = tp->snd_recover_prev;
3984 tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent);
3987 tp->t_badrxtwin = 0;
3995 * - There is no delayed ack timer in progress.
3996 * - Our last ack wasn't a 0-sized window. We never want to delay
3997 * the ack that opens up a 0-sized window.
3998 * - LRO wasn't used for this segment. We make sure by checking that the
4000 * - Delayed acks are enabled or this is a half-synchronized T/TCP
4002 * - The data being acked is less than a full segment (a stretch ack
4004 * - nsegs is 1 (if its more than that we received more than 1 ack).
4007 (((tp->t_flags & TF_RXWIN0SENT) == 0) && \
4008 ((tp->t_flags & TF_DELACK) == 0) && \
4009 ((bbr->bbr_segs_rcvd + nsegs) < tp->t_delayed_ack) && \
4010 (tp->t_delayed_ack || (tp->t_flags & TF_NEEDSYN)))
4025 * Walk the time-order transmitted list looking for an rsm that is
4029 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_tmap, r_tnext) {
4030 if (rsm->r_flags & BBR_ACKED) {
4050 TAILQ_FOREACH_REVERSE_FROM(prsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
4051 if (prsm->r_flags & (BBR_ACKED | BBR_HAS_FIN)) {
4072 * If reorder-fade is configured, then we track the last time we saw
4073 * re-ordering occur. If we reach the point where enough time as
4076 * Or if reorder-face is 0, then once we see reordering we consider
4080 * In the end if lro is non-zero we add the extra time for
4088 if (bbr->r_ctl.rc_reorder_ts) {
4089 if (bbr->r_ctl.rc_reorder_fade) {
4090 if (SEQ_GEQ(cts, bbr->r_ctl.rc_reorder_ts)) {
4091 lro = cts - bbr->r_ctl.rc_reorder_ts;
4103 if (lro > bbr->r_ctl.rc_reorder_fade) {
4105 bbr->r_ctl.rc_reorder_ts = 0;
4115 thresh = srtt + bbr->r_ctl.rc_pkt_delay;
4118 if (bbr->r_ctl.rc_reorder_shift)
4119 thresh += (srtt >> bbr->r_ctl.rc_reorder_shift);
4126 if ((bbr->rc_tp)->t_srtt == 0)
4129 t_rxtcur = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
4134 if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {
4135 thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND);
4142 * Return to the caller the amount of time in mico-seconds
4156 if (bbr->rc_tlp_threshold)
4157 thresh = srtt + (srtt / bbr->rc_tlp_threshold);
4160 maxseg = tp->t_maxseg - bbr->rc_last_options;
4162 len = rsm->r_end - rsm->r_start;
4169 * possible inter-packet delay (if any).
4174 idx = rsm->r_rtr_cnt - 1;
4175 nidx = prsm->r_rtr_cnt - 1;
4176 if (TSTMP_GEQ(rsm->r_tim_lastsent[nidx], prsm->r_tim_lastsent[idx])) {
4178 inter_gap = rsm->r_tim_lastsent[idx] - prsm->r_tim_lastsent[nidx];
4183 * Possibly compensate for delayed-ack.
4192 if (tp->t_srtt == 0)
4195 t_rxtcur = TICKS_2_USEC(tp->t_rxtcur);
4203 if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {
4204 thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND);
4222 f_rtt = get_filter_value_small(&bbr->r_ctl.rc_rttprop);
4223 if (get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) {
4225 if (bbr->rc_tp->t_srtt == 0)
4228 f_rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);
4231 * delayed-ack min
4237 /* Take the filter version or last measured pkt-rtt */
4241 if (bbr->r_ctl.rc_pkt_epoch_rtt) {
4242 srtt = bbr->r_ctl.rc_pkt_epoch_rtt;
4248 srtt = bbr->r_ctl.rc_last_rtt;
4250 if (bbr->rc_ack_was_delayed)
4251 srtt += bbr->r_ctl.rc_ack_hdwr_delay;
4253 srtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);
4271 if ((cts - rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]) >= thresh) {
4293 if (TAILQ_EMPTY(&bbr->r_ctl.rc_map)) {
4297 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
4302 if (tp->t_flags & TF_SENTFIN) {
4306 if (rsm->r_flags & BBR_ACKED) {
4315 idx = rsm->r_rtr_cnt - 1;
4316 if (SEQ_LEQ(cts, rsm->r_tim_lastsent[idx])) {
4322 ((rsm->r_dupack >= DUP_ACK_THRESHOLD) ||
4323 (rsm->r_flags & BBR_SACK_PASSED))) {
4324 if ((rsm->r_flags & BBR_MARKED_LOST) == 0) {
4325 rsm->r_flags |= BBR_MARKED_LOST;
4326 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;
4327 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;
4331 if ((rsm->r_end - rsm->r_start) == 0)
4352 * retransmissions, if so we will enter fast-recovery. The output
4358 if (bbr->rc_all_timers_stopped) {
4361 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {
4366 lost = bbr->r_ctl.rc_lost;
4367 if (bbr->r_state && (bbr->r_state != tp->t_state))
4370 if (bbr->r_ctl.rc_resend == NULL) {
4372 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);
4375 bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost));
4376 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RACK;
4385 nrsm->r_start = start;
4386 nrsm->r_end = rsm->r_end;
4387 nrsm->r_rtr_cnt = rsm->r_rtr_cnt;
4388 nrsm-> r_rtt_not_allowed = rsm->r_rtt_not_allowed;
4389 nrsm->r_flags = rsm->r_flags;
4391 nrsm->r_flags &= ~BBR_HAS_SYN;
4393 rsm->r_flags &= ~BBR_HAS_FIN;
4394 nrsm->r_dupack = rsm->r_dupack;
4395 nrsm->r_rtr_bytes = 0;
4396 nrsm->r_is_gain = rsm->r_is_gain;
4397 nrsm->r_is_drain = rsm->r_is_drain;
4398 nrsm->r_delivered = rsm->r_delivered;
4399 nrsm->r_ts_valid = rsm->r_ts_valid;
4400 nrsm->r_del_ack_ts = rsm->r_del_ack_ts;
4401 nrsm->r_del_time = rsm->r_del_time;
4402 nrsm->r_app_limited = rsm->r_app_limited;
4403 nrsm->r_first_sent_time = rsm->r_first_sent_time;
4404 nrsm->r_flight_at_send = rsm->r_flight_at_send;
4406 nrsm->r_bbr_state = rsm->r_bbr_state;
4407 for (idx = 0; idx < nrsm->r_rtr_cnt; idx++) {
4408 nrsm->r_tim_lastsent[idx] = rsm->r_tim_lastsent[idx];
4410 rsm->r_end = nrsm->r_start;
4411 idx = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs);
4414 if ((rsm->r_is_smallmap == 0) &&
4415 ((rsm->r_end - rsm->r_start) <= idx)) {
4416 bbr->r_ctl.rc_num_small_maps_alloced++;
4417 rsm->r_is_smallmap = 1;
4420 if ((nrsm->r_end - nrsm->r_start) <= idx) {
4421 bbr->r_ctl.rc_num_small_maps_alloced++;
4422 nrsm->r_is_smallmap = 1;
4445 if (l_rsm && (l_rsm->r_flags & BBR_ACKED)) {
4447 if ((l_rsm->r_end == start) ||
4448 (SEQ_LT(start, l_rsm->r_end) &&
4449 SEQ_GT(end, l_rsm->r_end))) {
4451 * map blk |------|
4452 * sack blk |------|
4454 * map blk |------|
4455 * sack blk |------|
4460 if (r_rsm && (r_rsm->r_flags & BBR_ACKED)) {
4462 if ((r_rsm->r_start == end) ||
4463 (SEQ_LT(start, r_rsm->r_start) &&
4464 SEQ_GT(end, r_rsm->r_start))) {
4466 * map blk |---------|
4467 * sack blk |----|
4469 * map blk |---------|
4470 * sack blk |-------|
4493 l_rsm->r_end = r_rsm->r_end;
4494 if (l_rsm->r_dupack < r_rsm->r_dupack)
4495 l_rsm->r_dupack = r_rsm->r_dupack;
4496 if (r_rsm->r_rtr_bytes)
4497 l_rsm->r_rtr_bytes += r_rsm->r_rtr_bytes;
4498 if (r_rsm->r_in_tmap) {
4500 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, r_rsm, r_tnext);
4502 if (r_rsm->r_app_limited)
4503 l_rsm->r_app_limited = r_rsm->r_app_limited;
4505 if (r_rsm->r_flags & BBR_HAS_FIN)
4506 l_rsm->r_flags |= BBR_HAS_FIN;
4507 if (r_rsm->r_flags & BBR_TLP)
4508 l_rsm->r_flags |= BBR_TLP;
4509 if (r_rsm->r_flags & BBR_RWND_COLLAPSED)
4510 l_rsm->r_flags |= BBR_RWND_COLLAPSED;
4511 if (r_rsm->r_flags & BBR_MARKED_LOST) {
4513 bbr->r_ctl.rc_lost_bytes -= r_rsm->r_end - r_rsm->r_start;
4515 TAILQ_REMOVE(&bbr->r_ctl.rc_map, r_rsm, r_next);
4516 if ((r_rsm->r_limit_type == 0) && (l_rsm->r_limit_type != 0)) {
4518 r_rsm->r_limit_type = l_rsm->r_limit_type;
4519 l_rsm->r_limit_type = 0;
4546 if (bbr->rc_all_timers_stopped) {
4549 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {
4555 return (-ETIMEDOUT); /* tcp_drop() */
4558 if (bbr->rc_in_persist) {
4561 if (bbr->r_state && (bbr->r_state != tp->t_state))
4564 maxseg = tp->t_maxseg - bbr->rc_last_options;
4570 avail = sbavail(&so->so_snd);
4572 if (out > tp->snd_wnd) {
4579 amm = avail - out;
4582 } else if ((amm < maxseg) && ((tp->t_flags & TF_NODELAY) == 0)) {
4583 /* not enough to fill a MTU and no-delay is off */
4586 /* Set the send-new override */
4587 if ((out + amm) <= tp->snd_wnd) {
4588 bbr->rc_tlp_new_data = 1;
4592 bbr->r_ctl.rc_tlp_seg_send_cnt = 0;
4593 bbr->r_ctl.rc_last_tlp_seq = tp->snd_max;
4594 bbr->r_ctl.rc_tlp_send = NULL;
4601 * Ok we need to arrange the last un-acked segment to be re-sent, or
4602 * optionally the first un-acked segment.
4605 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);
4606 if (rsm && (rsm->r_flags & (BBR_ACKED | BBR_HAS_FIN))) {
4617 TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
4618 if ((rsm->r_flags & BBR_RWND_COLLAPSED) == 0) {
4625 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
4630 if ((rsm->r_end - rsm->r_start) > maxseg) {
4641 * do the large send (BTLP :-) ).
4645 bbr_clone_rsm(bbr, nrsm, rsm, (rsm->r_end - maxseg));
4646 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
4647 if (rsm->r_in_tmap) {
4648 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
4649 nrsm->r_in_tmap = 1;
4651 rsm->r_flags &= (~BBR_HAS_FIN);
4655 bbr->r_ctl.rc_tlp_send = rsm;
4656 bbr->rc_tlp_rtx_out = 1;
4657 if (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) {
4658 bbr->r_ctl.rc_tlp_seg_send_cnt++;
4659 tp->t_rxtshift++;
4661 bbr->r_ctl.rc_last_tlp_seq = rsm->r_start;
4662 bbr->r_ctl.rc_tlp_seg_send_cnt = 1;
4665 if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) {
4671 bbr->rc_tlp_new_data = 0;
4672 bbr->r_ctl.rc_tlp_send = NULL;
4674 rsm->r_flags &= ~BBR_TLP;
4678 rsm->r_flags |= BBR_TLP;
4680 if (rsm && (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) &&
4681 (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend)) {
4689 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_TLP;
4704 if (bbr->rc_all_timers_stopped) {
4708 tp->t_flags &= ~TF_DELACK;
4709 tp->t_flags |= TF_ACKNOW;
4711 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_DELACK;
4716 * Here we send a KEEP-ALIVE like probe to the
4728 if (bbr->rc_all_timers_stopped) {
4731 if (bbr->rc_in_persist == 0)
4739 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_PERSIT;
4746 return (-ETIMEDOUT); /* tcp_drop() */
4754 if (tp->t_rxtshift >= V_tcp_retries &&
4755 (ticks - tp->t_rcvtime >= tcp_maxpersistidle ||
4756 ticks - tp->t_rcvtime >= TCP_REXMTVAL(tp) * tcp_totbackoff)) {
4759 return (-ETIMEDOUT); /* tcp_drop() */
4761 if ((sbavail(&bbr->rc_inp->inp_socket->so_snd) == 0) &&
4762 tp->snd_una == tp->snd_max) {
4771 if (tp->t_state > TCPS_CLOSE_WAIT &&
4772 (ticks - tp->t_rcvtime) >= TCPTV_PERSMAX) {
4775 return (-ETIMEDOUT); /* tcp_drop() */
4777 t_template = tcpip_maketemplate(bbr->rc_inp);
4779 tcp_respond(tp, t_template->tt_ipgen,
4780 &t_template->tt_t, (struct mbuf *)NULL,
4781 tp->rcv_nxt, tp->snd_una - 1, 0);
4783 if (tp->t_flags & TF_DELACK)
4784 tp->t_flags &= ~TF_DELACK;
4787 if (tp->t_rxtshift < V_tcp_retries)
4788 tp->t_rxtshift++;
4806 if (bbr->rc_all_timers_stopped) {
4809 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_KEEP;
4812 * Keep-alive timer went off; send something or drop connection if
4816 if (tp->t_state < TCPS_ESTABLISHED)
4818 if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) &&
4819 tp->t_state <= TCPS_CLOSING) {
4820 if (ticks - tp->t_rcvtime >= TP_KEEPIDLE(tp) + TP_MAXIDLE(tp))
4827 * number tp->snd_una-1 causes the transmitted zero-length
4835 tcp_respond(tp, t_template->tt_ipgen,
4836 &t_template->tt_t, (struct mbuf *)NULL,
4837 tp->rcv_nxt, tp->snd_una - 1, 0);
4846 return (-ETIMEDOUT); /* tcp_drop() */
4858 * un-acked.
4864 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
4865 cts = tcp_get_usecs(&bbr->rc_tv);
4866 lost = bbr->r_ctl.rc_lost;
4867 if (bbr->r_state && (bbr->r_state != tp->t_state))
4870 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
4871 if (rsm->r_flags & BBR_ACKED) {
4874 rsm->r_dupack = 0;
4875 if (rsm->r_in_tmap == 0) {
4876 /* We must re-add it back to the tlist */
4878 TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
4880 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, trsm, rsm, r_tnext);
4882 rsm->r_in_tmap = 1;
4884 old_flags = rsm->r_flags;
4885 rsm->r_flags |= BBR_RXT_CLEARED;
4886 rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS);
4889 if ((tp->t_state < TCPS_ESTABLISHED) &&
4890 (rsm->r_start == tp->snd_una)) {
4898 if ((rsm->r_flags & BBR_MARKED_LOST) == 0) {
4899 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;
4900 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;
4907 rsm->r_flags |= BBR_SACK_PASSED | BBR_MARKED_LOST;
4908 rsm->r_flags &= ~BBR_WAS_SACKPASS;
4917 rsm->r_flags |= BBR_MARKED_LOST;
4918 rsm->r_flags &= ~BBR_WAS_SACKPASS;
4919 rsm->r_flags &= ~BBR_SACK_PASSED;
4924 bbr->r_ctl.rc_resend = TAILQ_FIRST(&bbr->r_ctl.rc_map);
4925 /* Clear the count (we just un-acked them) */
4927 bbr->rc_tlp_new_data = 0;
4928 bbr->r_ctl.rc_tlp_seg_send_cnt = 0;
4930 bbr->r_ctl.rc_hptsi_agg_delay = 0;
4931 bbr->r_agg_early_set = 0;
4932 bbr->r_ctl.rc_agg_early = 0;
4933 bbr->rc_tlp_rtx_out = 0;
4934 bbr->r_ctl.rc_sacked = 0;
4935 bbr->r_ctl.rc_sacklast = NULL;
4936 bbr->r_timer_override = 1;
4937 bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost));
4941 * Re-transmit timeout! If we drop the PCB we will return 1, otherwise
4952 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RXT;
4953 if (bbr->rc_all_timers_stopped) {
4956 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
4957 (tp->snd_una == tp->snd_max)) {
4968 return (-ETIMEDOUT); /* tcp_drop() */
4971 if ((bbr->r_ctl.rc_resend == NULL) ||
4972 ((bbr->r_ctl.rc_resend->r_flags & BBR_RWND_COLLAPSED) == 0)) {
4979 tp->t_rxtshift++;
4981 if (tp->t_rxtshift > V_tcp_retries) {
4982 tp->t_rxtshift = V_tcp_retries;
4986 MPASS(tp->t_softerror >= 0);
4987 retval = tp->t_softerror ? -tp->t_softerror : -ETIMEDOUT;
4990 if (tp->t_state == TCPS_SYN_SENT) {
4995 tp->snd_cwnd = 1;
4996 } else if (tp->t_rxtshift == 1) {
5003 * End-to-End Network Path Properties" by Allman and Paxson
5006 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;
5007 if (!IN_RECOVERY(tp->t_flags)) {
5008 tp->snd_cwnd_prev = tp->snd_cwnd;
5009 tp->snd_ssthresh_prev = tp->snd_ssthresh;
5010 tp->snd_recover_prev = tp->snd_recover;
5011 tp->t_badrxtwin = ticks + (tp->t_srtt >> (TCP_RTT_SHIFT + 1));
5012 tp->t_flags |= TF_PREVVALID;
5014 tp->t_flags &= ~TF_PREVVALID;
5016 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;
5018 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;
5019 tp->t_flags &= ~TF_PREVVALID;
5022 if ((tp->t_state == TCPS_SYN_SENT) ||
5023 (tp->t_state == TCPS_SYN_RECEIVED))
5024 rexmt = USEC_2_TICKS(BBR_INITIAL_RTO) * tcp_backoff[tp->t_rxtshift];
5026 rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift];
5027 TCPT_RANGESET(tp->t_rxtcur, rexmt,
5028 MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms),
5029 MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000));
5038 isipv6 = (inp->inp_vflag & INP_IPV6) ? true : false;
5045 ((tp->t_state == TCPS_ESTABLISHED) ||
5046 (tp->t_state == TCPS_FIN_WAIT_1))) {
5049 * 1448 -> 1188 -> 524) should be given 2 chances to recover
5050 * before further clamping down. 'tp->t_rxtshift % 2 == 0'
5053 if (((tp->t_flags2 & (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) ==
5055 (tp->t_rxtshift >= 2 && tp->t_rxtshift < 6 &&
5056 tp->t_rxtshift % 2 == 0)) {
5058 * Enter Path MTU Black-hole Detection mechanism: -
5059 * Disable Path MTU Discovery (IP "DF" bit). -
5063 if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) == 0) {
5068 tp->t_flags2 |= TF2_PLPMTU_BLACKHOLE;
5070 tp->t_pmtud_saved_maxseg = tp->t_maxseg;
5077 isipv6 = bbr->r_is_v6;
5079 tp->t_maxseg > V_tcp_v6pmtud_blackhole_mss) {
5081 tp->t_maxseg = V_tcp_v6pmtud_blackhole_mss;
5085 tp->t_maxseg = V_tcp_v6mssdflt;
5090 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
5098 if (tp->t_maxseg > V_tcp_pmtud_blackhole_mss) {
5100 tp->t_maxseg = V_tcp_pmtud_blackhole_mss;
5104 tp->t_maxseg = V_tcp_mssdflt;
5109 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
5122 if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) &&
5123 (tp->t_rxtshift >= 6)) {
5124 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
5125 tp->t_flags2 &= ~TF2_PLPMTU_BLACKHOLE;
5126 tp->t_maxseg = tp->t_pmtud_saved_maxseg;
5127 if (tp->t_maxseg < V_tcp_mssdflt) {
5133 tp->t_flags2 |= TF2_PROC_SACK_PROHIBIT;
5135 tp->t_flags2 &= ~TF2_PROC_SACK_PROHIBIT;
5143 * third SYN to work-around some broken terminal servers (most of
5146 * unknown-to-them TCP options.
5148 if (tcp_rexmit_drop_options && (tp->t_state == TCPS_SYN_SENT) &&
5149 (tp->t_rxtshift == 3))
5150 tp->t_flags &= ~(TF_REQ_SCALE | TF_REQ_TSTMP | TF_SACK_PERMIT);
5157 if (tp->t_rxtshift > TCP_MAXRXTSHIFT / 4) {
5159 if (bbr->r_is_v6)
5164 tp->t_rttvar += (tp->t_srtt >> TCP_RTT_SHIFT);
5165 tp->t_srtt = 0;
5167 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
5168 tp->snd_recover = tp->snd_max;
5169 tp->t_flags |= TF_ACKNOW;
5170 tp->t_rtttime = 0;
5179 int32_t timers = (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK);
5184 if (tp->t_state == TCPS_LISTEN) {
5186 if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)
5190 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {
5193 if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
5194 ret = -1;
5199 ret = -2;
5207 left = bbr->r_ctl.rc_timer_exp - cts;
5208 ret = -3;
5213 bbr->rc_tmr_stopped = 0;
5214 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_MASK;
5220 bbr->r_ctl.rc_tlp_rxt_last_time = cts;
5223 bbr->r_ctl.rc_tlp_rxt_last_time = cts;
5226 bbr->r_ctl.rc_tlp_rxt_last_time = cts;
5238 if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
5241 if (tcp_in_hpts(bbr->rc_tp) &&
5242 (bbr->rc_timer_first == 1)) {
5249 tcp_hpts_remove(bbr->rc_tp);
5250 if (bbr->r_ctl.rc_last_delay_val) {
5254 if (TSTMP_GT(cts, bbr->rc_pacer_started))
5255 time_since_send = cts - bbr->rc_pacer_started;
5258 if (bbr->r_ctl.rc_last_delay_val > time_since_send) {
5260 bbr->r_ctl.rc_last_delay_val -= time_since_send;
5262 bbr->r_ctl.rc_last_delay_val = 0;
5264 bbr->rc_pacer_started = cts;
5267 bbr->rc_timer_first = 0;
5269 bbr->rc_tmr_stopped = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK;
5270 bbr->r_ctl.rc_hpts_flags &= ~(PACE_TMR_MASK);
5279 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
5280 bbr->rc_all_timers_stopped = 1;
5292 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
5295 return(rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]);
5304 rsm->r_rtr_cnt++;
5305 rsm->r_dupack = 0;
5306 if (rsm->r_rtr_cnt > BBR_NUM_OF_RETRANS) {
5307 rsm->r_rtr_cnt = BBR_NUM_OF_RETRANS;
5308 rsm->r_flags |= BBR_OVERMAX;
5310 if (rsm->r_flags & BBR_RWND_COLLAPSED) {
5312 rsm->r_flags &= ~BBR_RWND_COLLAPSED;
5314 if (rsm->r_flags & BBR_MARKED_LOST) {
5316 rsm->r_flags &= ~BBR_MARKED_LOST;
5317 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
5319 if (rsm->r_flags & BBR_RXT_CLEARED) {
5328 rsm->r_flags &= ~BBR_RXT_CLEARED;
5330 if ((rsm->r_rtr_cnt > 1) && ((rsm->r_flags & BBR_TLP) == 0)) {
5331 bbr->r_ctl.rc_holes_rxt += (rsm->r_end - rsm->r_start);
5332 rsm->r_rtr_bytes += (rsm->r_end - rsm->r_start);
5334 idx = rsm->r_rtr_cnt - 1;
5335 rsm->r_tim_lastsent[idx] = cts;
5336 rsm->r_pacing_delay = pacing_time;
5337 rsm->r_delivered = bbr->r_ctl.rc_delivered;
5338 rsm->r_ts_valid = bbr->rc_ts_valid;
5339 if (bbr->rc_ts_valid)
5340 rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts;
5341 if (bbr->r_ctl.r_app_limited_until)
5342 rsm->r_app_limited = 1;
5344 rsm->r_app_limited = 0;
5345 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)
5346 rsm->r_bbr_state = bbr_state_val(bbr);
5348 rsm->r_bbr_state = 8;
5349 if (rsm->r_flags & BBR_ACKED) {
5353 old_flags = rsm->r_flags;
5354 rsm->r_flags &= ~BBR_ACKED;
5356 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
5357 if (bbr->r_ctl.rc_sacked == 0)
5358 bbr->r_ctl.rc_sacklast = NULL;
5360 if (rsm->r_in_tmap) {
5361 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
5363 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
5364 rsm->r_in_tmap = 1;
5365 if (rsm->r_flags & BBR_SACK_PASSED) {
5367 rsm->r_flags &= ~BBR_SACK_PASSED;
5368 rsm->r_flags |= BBR_WAS_SACKPASS;
5370 rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts);
5371 rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp,
5372 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
5373 bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next);
5374 if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) {
5375 rsm->r_is_gain = 1;
5376 rsm->r_is_drain = 0;
5377 } else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) {
5378 rsm->r_is_drain = 1;
5379 rsm->r_is_gain = 0;
5381 rsm->r_is_drain = 0;
5382 rsm->r_is_gain = 0;
5384 rsm->r_del_time = bbr->r_ctl.rc_del_time; /* TEMP GOOGLE CODE */
5398 * We (re-)transmitted starting at rsm->r_start for some length
5406 c_end = rsm->r_start + len;
5407 if (SEQ_GEQ(c_end, rsm->r_end)) {
5413 if (c_end == rsm->r_end) {
5420 act_len = rsm->r_end - rsm->r_start;
5421 *lenp = (len - act_len);
5422 return (rsm->r_end);
5443 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
5444 nrsm->r_dupack = 0;
5445 if (rsm->r_in_tmap) {
5446 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
5447 nrsm->r_in_tmap = 1;
5449 rsm->r_flags &= (~BBR_HAS_FIN);
5474 if (get_filter_value(&bbr->r_ctl.rc_delrate) >= act_rate) {
5478 bbr->skip_gain = 1;
5479 bbr->gain_is_limited = 0;
5480 red = get_filter_value(&bbr->r_ctl.rc_delrate) - act_rate;
5482 filter_reduce_by(&bbr->r_ctl.rc_delrate, red, cts);
5485 bbr->skip_gain = 0;
5486 bbr->gain_is_limited = 1;
5494 int error, rate = -1;
5496 if (bbr->r_ctl.crte == NULL)
5498 if ((bbr->rc_inp->inp_route.ro_nh == NULL) ||
5499 (bbr->rc_inp->inp_route.ro_nh->nh_ifp == NULL)) {
5501 /* Clear the way for a re-attempt */
5502 bbr->bbr_attempt_hdwr_pace = 0;
5504 bbr->gain_is_limited = 0;
5505 bbr->skip_gain = 0;
5506 bbr->bbr_hdrw_pacing = 0;
5507 counter_u64_add(bbr_flows_whdwr_pacing, -1);
5513 nrte = tcp_chg_pacing_rate(bbr->r_ctl.crte,
5514 bbr->rc_tp,
5515 bbr->rc_inp->inp_route.ro_nh->nh_ifp,
5522 if (nrte != bbr->r_ctl.crte) {
5523 bbr->r_ctl.crte = nrte;
5526 if (bbr->r_ctl.crte->rate < rate) {
5529 bbr->r_ctl.crte->rate, rate);
5532 bbr->gain_is_limited = 0;
5533 bbr->skip_gain = 0;
5538 bbr->gain_is_limited = 0;
5539 bbr->skip_gain = 0;
5540 bbr->bbr_hdrw_pacing = 0;
5543 bbr->r_ctl.crte->ptbl->rs_ifp,
5545 bbr->r_ctl.crte->rate,
5563 if ((bbr->bbr_hdrw_pacing == 0) ||
5564 (IN_RECOVERY(bbr->rc_tp->t_flags)) ||
5565 (bbr->r_ctl.crte == NULL))
5567 if (bbr->hw_pacing_set == 0) {
5575 rlp = bbr->r_ctl.crte;
5576 if (bbr->rc_tp->t_maxseg > bbr->rc_last_options)
5577 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
5579 maxseg = BBR_MIN_SEG - bbr->rc_last_options;
5587 bbr->r_ctl.rc_pace_max_segs, cts, 1);
5588 hdwr_delay = bbr->r_ctl.rc_pace_max_segs / maxseg;
5589 hdwr_delay *= rlp->time_between;
5591 delta = cur_delay - hdwr_delay;
5595 (bbr->r_ctl.rc_pace_max_segs / maxseg),
5598 (delta < (max(rlp->time_between,
5599 bbr->r_ctl.bbr_hptsi_segments_delay_tar)))) {
5613 seg_sz = max(((cur_delay + rlp->time_between)/rlp->time_between),
5614 (bbr->r_ctl.rc_pace_max_segs/maxseg));
5617 (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) {
5623 seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg;
5637 seg_sz = bbr->r_ctl.rc_pace_max_segs * bbr_hdwr_pace_adjust;
5639 (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) {
5645 seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg;
5659 seg_sz = bbr->r_ctl.rc_pace_max_segs;
5661 if (seg_sz > bbr->r_ctl.rc_pace_max_segs)
5664 new_tso = bbr->r_ctl.rc_pace_max_segs;
5665 if (new_tso >= (PACE_MAX_IP_BYTES-maxseg))
5666 new_tso = PACE_MAX_IP_BYTES - maxseg;
5668 if (new_tso != bbr->r_ctl.rc_pace_max_segs) {
5669 bbr_log_type_tsosize(bbr, cts, new_tso, 0, bbr->r_ctl.rc_pace_max_segs, maxseg, 0);
5670 bbr->r_ctl.rc_pace_max_segs = new_tso;
5697 * a full iwnd just like new-reno/cubic.
5702 * if ( bw <= per-tcb-cross-over)
5704 * can send in goal-time seconds.
5710 * if (tso > per-tcb-max)
5711 * tso = per-tcb-max
5713 * tso = max-tso (64k/mss)
5715 * goal_tso = bw / per-tcb-divsor
5716 * seg = (goal_tso + mss-1)/mss
5719 * if (tso < per-tcb-floor)
5720 * tso = per-tcb-floor
5721 * if (tso > per-tcb-utter_max)
5722 * tso = per-tcb-utter_max
5724 * Note the default per-tcb-divisor is 1000 (same as google).
5728 * cross-over = 23,168,000 bps
5729 * goal-time = 18000
5730 * per-tcb-max = 2
5731 * per-tcb-divisor = 1000
5732 * per-tcb-floor = 1
5741 if (bbr->rc_tp->t_maxseg > bbr->rc_last_options) {
5742 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
5744 maxseg = BBR_MIN_SEG - bbr->rc_last_options;
5746 old_tso = bbr->r_ctl.rc_pace_max_segs;
5747 if (bbr->rc_past_init_win == 0) {
5753 if (bbr->rc_use_google)
5754 bbr->r_ctl.rc_pace_max_segs = ((bbr->rc_tp->t_maxseg - bbr->rc_last_options) * 2);
5755 else if (bbr->bbr_init_win_cheat)
5756 bbr->r_ctl.rc_pace_max_segs = bbr_initial_cwnd(bbr, bbr->rc_tp);
5758 bbr->r_ctl.rc_pace_max_segs = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
5759 if (bbr->r_ctl.rc_pace_min_segs != bbr->rc_tp->t_maxseg)
5760 bbr->r_ctl.rc_pace_min_segs = bbr->rc_tp->t_maxseg;
5761 if (bbr->r_ctl.rc_pace_max_segs == 0) {
5762 bbr->r_ctl.rc_pace_max_segs = maxseg;
5764 bbr_log_type_tsosize(bbr, cts, bbr->r_ctl.rc_pace_max_segs, tls_seg, old_tso, maxseg, 0);
5774 if (IN_RECOVERY(bbr->rc_tp->t_flags) &&
5775 (bbr->rc_use_google == 0)) {
5778 } else if (bbr->rc_use_google) {
5782 if (bbr->r_ctl.rc_bbr_hptsi_gain != BBR_UNIT) {
5783 bw *= bbr->r_ctl.rc_bbr_hptsi_gain;
5797 } else if (bbr->rc_no_pacing) {
5799 } else if (bw <= bbr->r_ctl.bbr_cross_over) {
5806 tso_len = bbr_get_pacing_length(bbr, BBR_UNIT, bbr->r_ctl.bbr_hptsi_segments_delay_tar, bw);
5809 if (new_tso > bbr->r_ctl.bbr_hptsi_segments_max)
5810 new_tso = bbr->r_ctl.bbr_hptsi_segments_max;
5843 bw /= bbr->r_ctl.bbr_hptsi_per_second;
5850 if (new_tso < (bbr->r_ctl.bbr_hptsi_segments_max * maxseg))
5851 new_tso = (bbr->r_ctl.bbr_hptsi_segments_max * maxseg);
5853 if (bbr->r_ctl.bbr_hptsi_segments_floor && (new_tso < (maxseg * bbr->r_ctl.bbr_hptsi_segments_floor)))
5854 new_tso = maxseg * bbr->r_ctl.bbr_hptsi_segments_floor;
5858 if (bbr->r_ctl.bbr_utter_max && (new_tso > (bbr->r_ctl.bbr_utter_max * maxseg))) {
5859 new_tso = bbr->r_ctl.bbr_utter_max * maxseg;
5864 bbr->r_ctl.rc_pace_max_segs = new_tso;
5899 * We don't log errors -- we could but snd_max does not
5912 snd_una = tp->snd_una;
5920 if ((th_flags & TH_SYN) && (tp->iss == seq_out))
5926 /* Are sending an old segment to induce an ack (keep-alive)? */
5935 len = end - seq_out;
5937 snd_max = tp->snd_max;
5942 pacing_time = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, len, cts, 1);
5950 rsm->r_flags = 0;
5952 rsm->r_flags |= BBR_HAS_SYN;
5954 rsm->r_flags |= BBR_HAS_FIN;
5955 rsm->r_tim_lastsent[0] = cts;
5956 rsm->r_rtr_cnt = 1;
5957 rsm->r_rtr_bytes = 0;
5958 rsm->r_start = seq_out;
5959 rsm->r_end = rsm->r_start + len;
5960 rsm->r_dupack = 0;
5961 rsm->r_delivered = bbr->r_ctl.rc_delivered;
5962 rsm->r_pacing_delay = pacing_time;
5963 rsm->r_ts_valid = bbr->rc_ts_valid;
5964 if (bbr->rc_ts_valid)
5965 rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts;
5966 rsm->r_del_time = bbr->r_ctl.rc_del_time;
5967 if (bbr->r_ctl.r_app_limited_until)
5968 rsm->r_app_limited = 1;
5970 rsm->r_app_limited = 0;
5971 rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts);
5972 rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp,
5973 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
5978 rsm->r_flight_at_send += len;
5979 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next);
5980 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
5981 rsm->r_in_tmap = 1;
5982 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)
5983 rsm->r_bbr_state = bbr_state_val(bbr);
5985 rsm->r_bbr_state = 8;
5986 if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) {
5987 rsm->r_is_gain = 1;
5988 rsm->r_is_drain = 0;
5989 } else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) {
5990 rsm->r_is_drain = 1;
5991 rsm->r_is_gain = 0;
5993 rsm->r_is_drain = 0;
5994 rsm->r_is_gain = 0;
6002 if (hintrsm && (hintrsm->r_start == seq_out)) {
6005 } else if (bbr->r_ctl.rc_next) {
6007 rsm = bbr->r_ctl.rc_next;
6012 if ((rsm) && (rsm->r_start == seq_out)) {
6025 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
6026 if (rsm->r_start == seq_out) {
6028 bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next);
6035 if (SEQ_GEQ(seq_out, rsm->r_start) && SEQ_LT(seq_out, rsm->r_end)) {
6051 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
6052 if (rsm->r_in_tmap) {
6053 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
6054 nrsm->r_in_tmap = 1;
6056 rsm->r_flags &= (~BBR_HAS_FIN);
6067 if (seq_out == tp->snd_max) {
6069 } else if (SEQ_LT(seq_out, tp->snd_max)) {
6071 printf("seq_out:%u len:%d snd_una:%u snd_max:%u -- but rsm not found?\n",
6072 seq_out, len, tp->snd_una, tp->snd_max);
6074 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
6076 rsm, rsm->r_start, rsm->r_end);
6085 * Hmm beyond sndmax? (only if we are using the new rtt-pack
6089 seq_out, len, tp->snd_max, tp);
6098 * Collapse timeout back the cum-ack moved.
6100 tp->t_rxtshift = 0;
6101 tp->t_softerror = 0;
6107 bbr->rtt_valid = 1;
6108 bbr->r_ctl.cur_rtt = rtt_usecs;
6109 bbr->r_ctl.ts_in = tsin;
6111 bbr->r_ctl.cur_rtt_send_time = rsm_send_time;
6119 * 1) The timestamp we started observing cum-acks (bbr->r_ctl.bbr_ts_check_tstmp).
6120 * 2) Our timestamp indicating when we sent that packet (bbr->r_ctl.rsm->bbr_ts_check_our_cts).
6121 * 3) The current timestamp that just came in (bbr->r_ctl.last_inbound_ts)
6122 * 4) The time that the packet that generated that ack was sent (bbr->r_ctl.cur_rtt_send_time)
6126 * delta = bbr->r_ctl.cur_rtt_send_time - bbr->r_ctl.bbr_ts_check_our_cts
6130 * peer_delta = bbr->r_ctl.last_inbound_ts - bbr->r_ctl.bbr_ts_check_tstmp
6137 * out time-slices of the actual b/w. This would mean we could not use
6141 * timestamp difference which we will store in bbr->r_ctl.bbr_peer_tsratio.
6155 delta = bbr->r_ctl.cur_rtt_send_time - bbr->r_ctl.bbr_ts_check_our_cts;
6165 peer_delta = bbr->r_ctl.last_inbound_ts - bbr->r_ctl.bbr_ts_check_tstmp;
6184 * then we saw then in micro-seconds.
6187 /* well it looks like the peer is a micro-second clock. */
6188 bbr->rc_ts_clock_set = 1;
6189 bbr->r_ctl.bbr_peer_tsratio = 1;
6191 bbr->rc_ts_cant_be_used = 1;
6192 bbr->rc_ts_clock_set = 1;
6197 bbr->rc_ts_clock_set = 1;
6202 bbr->r_ctl.bbr_peer_tsratio = 1;
6209 bbr->r_ctl.bbr_peer_tsratio = 10;
6216 bbr->r_ctl.bbr_peer_tsratio = 100;
6223 bbr->r_ctl.bbr_peer_tsratio = 1000;
6230 bbr->r_ctl.bbr_peer_tsratio = 10000;
6234 bbr->rc_ts_cant_be_used = 1;
6235 bbr->r_ctl.bbr_peer_tsratio = 0;
6240 * Collect new round-trip time estimate
6250 if (bbr->rtt_valid == 0)
6254 rtt = bbr->r_ctl.cur_rtt;
6255 tsin = bbr->r_ctl.ts_in;
6256 if (bbr->rc_prtt_set_ts) {
6265 * and when they enter probe-rtt they update the
6270 bbr->rc_prtt_set_ts = 0;
6271 rtt_prop = get_filter_value_small(&bbr->r_ctl.rc_rttprop);
6273 filter_increase_by_small(&bbr->r_ctl.rc_rttprop, (rtt - rtt_prop), cts);
6275 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
6278 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_PATHRTT, imax(0, rtt));
6280 if (bbr->rc_ack_was_delayed)
6281 rtt += bbr->r_ctl.rc_ack_hdwr_delay;
6283 if (rtt < bbr->r_ctl.rc_lowest_rtt)
6284 bbr->r_ctl.rc_lowest_rtt = rtt;
6286 if (bbr->r_init_rtt) {
6288 * The initial rtt is not-trusted, nuke it and lets get
6291 bbr->r_init_rtt = 0;
6292 tp->t_srtt = 0;
6294 if ((bbr->rc_ts_clock_set == 0) && bbr->rc_ts_valid) {
6299 * series of cum-ack's to determine
6302 if (bbr->rc_ack_is_cumack) {
6303 if (bbr->rc_ts_data_set) {
6307 bbr->rc_ts_data_set = 1;
6308 bbr->r_ctl.bbr_ts_check_tstmp = bbr->r_ctl.last_inbound_ts;
6309 bbr->r_ctl.bbr_ts_check_our_cts = bbr->r_ctl.cur_rtt_send_time;
6316 bbr->rc_ts_data_set = 0;
6320 rtt_ticks = USEC_2_TICKS((rtt + (USECS_IN_MSEC - 1)));
6321 if (tp->t_srtt != 0) {
6330 delta = ((rtt_ticks - 1) << TCP_DELTA_SHIFT)
6331 - (tp->t_srtt >> (TCP_RTT_SHIFT - TCP_DELTA_SHIFT));
6333 tp->t_srtt += delta;
6334 if (tp->t_srtt <= 0)
6335 tp->t_srtt = 1;
6345 * wired-in beta.
6348 delta = -delta;
6349 delta -= tp->t_rttvar >> (TCP_RTTVAR_SHIFT - TCP_DELTA_SHIFT);
6350 tp->t_rttvar += delta;
6351 if (tp->t_rttvar <= 0)
6352 tp->t_rttvar = 1;
6355 * No rtt measurement yet - use the unsmoothed rtt. Set the
6359 tp->t_srtt = rtt_ticks << TCP_RTT_SHIFT;
6360 tp->t_rttvar = rtt_ticks << (TCP_RTTVAR_SHIFT - 1);
6363 if (tp->t_rttupdated < UCHAR_MAX)
6364 tp->t_rttupdated++;
6366 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, rtt_ticks));
6372 * tick of rounding and 1 extra tick because of +-1/2 tick
6378 TCPT_RANGESET(tp->t_rxtcur, TCP_REXMTVAL(tp),
6379 max(MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms), rtt_ticks + 2),
6380 MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000));
6389 tp->t_softerror = 0;
6390 rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);
6391 if (bbr->r_ctl.bbr_smallest_srtt_this_state > rtt)
6392 bbr->r_ctl.bbr_smallest_srtt_this_state = rtt;
6398 bbr->r_ctl.rc_rtt_shrinks = cts;
6400 (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) &&
6401 ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) {
6403 * We should enter probe-rtt its been too long
6416 if (bbr->r_ctl.rc_bbr_cur_del_rate == 0) {
6422 if (bbr->r_ctl.r_measurement_count < 0xffffffff)
6423 bbr->r_ctl.r_measurement_count++;
6424 orig_bw = get_filter_value(&bbr->r_ctl.rc_delrate);
6425 apply_filter_max(&bbr->r_ctl.rc_delrate, bbr->r_ctl.rc_bbr_cur_del_rate, bbr->r_ctl.rc_pkt_epoch);
6427 (uint32_t)get_filter_value(&bbr->r_ctl.rc_delrate),
6430 (orig_bw != get_filter_value(&bbr->r_ctl.rc_delrate))) {
6431 if (bbr->bbr_hdrw_pacing) {
6440 if (bbr->r_recovery_bw) {
6444 } else if ((orig_bw == 0) && get_filter_value(&bbr->r_ctl.rc_delrate))
6451 if (bbr->rc_in_persist == 0) {
6457 if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time))
6458 tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time);
6466 delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered);
6476 * can see in the trace viewer if it gets over-ridden.
6478 if (rsm->r_ts_valid &&
6479 bbr->rc_ts_valid &&
6480 bbr->rc_ts_clock_set &&
6481 (bbr->rc_ts_cant_be_used == 0) &&
6482 bbr->rc_use_ts_limit) {
6483 ts_diff = max((bbr->r_ctl.last_inbound_ts - rsm->r_del_ack_ts), 1);
6484 ts_diff *= bbr->r_ctl.bbr_peer_tsratio;
6490 bbr->r_ctl.last_inbound_ts,
6491 rsm->r_del_ack_ts, 0,
6501 if ((bbr->ts_can_raise) &&
6521 if (rsm->r_first_sent_time &&
6522 TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) {
6535 sbw = (uint64_t)(rsm->r_flight_at_send);
6537 sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time;
6538 sti += rsm->r_pacing_delay;
6546 rsm->r_first_sent_time, 0, (sbw >> 32),
6552 bbr->r_ctl.rc_bbr_cur_del_rate = bw;
6553 if ((rsm->r_app_limited == 0) ||
6554 (bw > get_filter_value(&bbr->r_ctl.rc_delrate))) {
6557 0, 0, 0, 0, bbr->r_ctl.rc_del_time, rsm->r_del_time);
6565 if (bbr->rc_in_persist == 0) {
6572 if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time))
6573 tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time);
6581 delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered);
6585 if (tim < bbr->r_ctl.rc_lowest_rtt) {
6587 tim, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0);
6593 * can see in the trace viewer if it gets over-ridden.
6595 bbr->r_ctl.rc_bbr_cur_del_rate = bw;
6597 if (rsm->r_first_sent_time &&
6598 TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) {
6611 sbw = (uint64_t)(rsm->r_flight_at_send);
6613 sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time;
6614 sti += rsm->r_pacing_delay;
6622 rsm->r_first_sent_time, 0, (sbw >> 32),
6627 (sti < bbr->r_ctl.rc_lowest_rtt)) {
6629 (uint32_t)sti, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0);
6634 bbr->r_ctl.rc_bbr_cur_del_rate = bw;
6636 ((rsm->r_app_limited == 0) ||
6637 (bw > get_filter_value(&bbr->r_ctl.rc_delrate)))) {
6640 0, 0, 0, 0, bbr->r_ctl.rc_del_time, rsm->r_del_time);
6652 bbr->r_ctl.rc_delivered += (rsm->r_end - rsm->r_start);
6653 bbr->r_ctl.rc_del_time = cts;
6661 if ((bbr->rc_use_google == 0) &&
6674 if ((bbr_no_retran && bbr->rc_use_google) &&
6678 * We only do measurements in google mode
6684 tcp_bbr_xmit_timer(bbr, rtt, rsm_send_time, rsm->r_start, tsin);
6686 bbr->rc_ack_is_cumack = 1;
6688 bbr->rc_ack_is_cumack = 0;
6695 * going into probe-rtt (we were seeing cases where that
6704 bbr_log_type_bbrrttprop(bbr, rtt, rsm->r_end, uts, cts,
6705 match, rsm->r_start, rsm->r_flags);
6706 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
6709 * The RTT-prop moved, reset the target (may be a
6713 if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT)
6720 if ((bbr->rc_use_google == 0) &&
6729 if (bbr->r_ctl.r_app_limited_until &&
6730 (bbr->r_ctl.rc_delivered >= bbr->r_ctl.r_app_limited_until)) {
6731 /* We are no longer app-limited */
6732 bbr->r_ctl.r_app_limited_until = 0;
6734 if (bbr->rc_use_google) {
6751 msec = cts - (MS_IN_USEC * sec);
6766 if ((rsm->r_flags & BBR_ACKED) ||
6767 (rsm->r_flags & BBR_WAS_RENEGED) ||
6768 (rsm->r_flags & BBR_RXT_CLEARED)) {
6772 if (rsm->r_rtt_not_allowed) {
6776 if (rsm->r_rtr_cnt == 1) {
6780 if (TSTMP_GT(cts, rsm->r_tim_lastsent[0]))
6781 t = cts - rsm->r_tim_lastsent[0];
6784 bbr->r_ctl.rc_last_rtt = t;
6785 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0,
6786 BBR_RTT_BY_EXACTMATCH, rsm->r_tim_lastsent[0], ack_type, to);
6791 (bbr->rc_use_google == 1) &&
6793 (to->to_flags & TOF_TS) &&
6794 (to->to_tsecr != 0)) {
6795 t = tcp_tv_to_mssectick(&bbr->rc_tv) - to->to_tsecr;
6799 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0,
6801 rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)],
6805 uts = bbr_ts_convert(to->to_tsecr);
6806 if ((to->to_flags & TOF_TS) &&
6807 (to->to_tsecr != 0) &&
6809 ((rsm->r_flags & BBR_OVERMAX) == 0)) {
6817 for (i = 0; i < rsm->r_rtr_cnt; i++) {
6818 if ((SEQ_GEQ(uts, (rsm->r_tim_lastsent[i] - fudge))) &&
6819 (SEQ_LEQ(uts, (rsm->r_tim_lastsent[i] + fudge)))) {
6820 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))
6821 t = cts - rsm->r_tim_lastsent[i];
6824 bbr->r_ctl.rc_last_rtt = t;
6825 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_TSMATCHING,
6826 rsm->r_tim_lastsent[i], ack_type, to);
6827 if ((i + 1) < rsm->r_rtr_cnt) {
6830 } else if (rsm->r_flags & BBR_TLP) {
6831 bbr->rc_tlp_rtx_out = 0;
6841 * time-stamp since its not there or the time the peer last
6842 * received a segment that moved forward its cum-ack point.
6848 i = rsm->r_rtr_cnt - 1;
6849 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))
6850 t = cts - rsm->r_tim_lastsent[i];
6853 if (t < bbr->r_ctl.rc_lowest_rtt) {
6859 * the rack-draft.
6865 if ((rsm->r_flags & BBR_OVERMAX) == 0) {
6867 if (rsm->r_rtr_cnt == 1)
6868 panic("rsm:%p bbr:%p rsm has overmax and only 1 retranmit flags:%x?", rsm, bbr, rsm->r_flags);
6870 i = rsm->r_rtr_cnt - 2;
6871 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))
6872 t = cts - rsm->r_tim_lastsent[i];
6875 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_EARLIER_RET,
6876 rsm->r_tim_lastsent[i], ack_type, to);
6884 bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts,
6892 if (rsm->r_flags & BBR_TLP)
6893 bbr->rc_tlp_rtx_out = 0;
6894 if ((rsm->r_flags & BBR_OVERMAX) == 0)
6895 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts,
6898 bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts,
6915 TAILQ_FOREACH_REVERSE_FROM(nrsm, &bbr->r_ctl.rc_tmap,
6921 if (nrsm->r_flags & BBR_ACKED) {
6925 if (nrsm->r_flags & BBR_SACK_PASSED) {
6934 nrsm->r_flags |= BBR_SACK_PASSED;
6935 if (((nrsm->r_flags & BBR_MARKED_LOST) == 0) &&
6936 bbr_is_lost(bbr, nrsm, bbr->r_ctl.rc_rcvtime)) {
6937 bbr->r_ctl.rc_lost += nrsm->r_end - nrsm->r_start;
6938 bbr->r_ctl.rc_lost_bytes += nrsm->r_end - nrsm->r_start;
6939 nrsm->r_flags |= BBR_MARKED_LOST;
6941 nrsm->r_flags &= ~BBR_WAS_SACKPASS;
6959 start = sack->start;
6960 end = sack->end;
6966 if (rsm && SEQ_LT(start, rsm->r_start)) {
6968 TAILQ_FOREACH_REVERSE_FROM(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
6969 if (SEQ_GEQ(start, rsm->r_start) &&
6970 SEQ_LT(start, rsm->r_end)) {
6981 TAILQ_FOREACH_FROM(rsm, &bbr->r_ctl.rc_map, r_next) {
6982 if (SEQ_GEQ(start, rsm->r_start) &&
6983 SEQ_LT(start, rsm->r_end)) {
6995 if (tp->t_flags & TF_SENTFIN) {
7000 nrsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);
7001 if (nrsm && (nrsm->r_end + 1) == tp->snd_max) {
7005 nrsm->r_end++;
7024 if (rsm->r_start != start) {
7038 sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk);
7042 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
7043 if (rsm->r_in_tmap) {
7044 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
7045 nrsm->r_in_tmap = 1;
7047 rsm->r_flags &= (~BBR_HAS_FIN);
7050 if (SEQ_GEQ(end, rsm->r_end)) {
7055 if ((rsm->r_flags & BBR_ACKED) == 0) {
7057 changed += (rsm->r_end - rsm->r_start);
7058 bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);
7060 if (rsm->r_flags & BBR_MARKED_LOST) {
7061 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7064 if (rsm->r_flags & BBR_SACK_PASSED) {
7066 bbr->r_ctl.rc_reorder_ts = cts;
7067 if (rsm->r_flags & BBR_MARKED_LOST) {
7068 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7069 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7071 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7074 rsm->r_flags |= BBR_ACKED;
7075 rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST);
7076 if (rsm->r_in_tmap) {
7077 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7078 rsm->r_in_tmap = 0;
7082 if (end == rsm->r_end) {
7083 /* This block only - done */
7087 start = rsm->r_end;
7093 if (rsm->r_flags & BBR_ACKED) {
7108 sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk);
7114 rsm->r_flags &= (~BBR_HAS_FIN);
7115 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
7116 if (rsm->r_in_tmap) {
7117 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
7118 nrsm->r_in_tmap = 1;
7120 nrsm->r_dupack = 0;
7123 changed += (rsm->r_end - rsm->r_start);
7124 bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);
7127 if (rsm->r_flags & BBR_MARKED_LOST) {
7128 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7130 if (rsm->r_flags & BBR_SACK_PASSED) {
7132 bbr->r_ctl.rc_reorder_ts = cts;
7133 if (rsm->r_flags & BBR_MARKED_LOST) {
7134 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7135 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7137 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7140 rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST);
7141 rsm->r_flags |= BBR_ACKED;
7142 if (rsm->r_in_tmap) {
7143 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7144 rsm->r_in_tmap = 0;
7147 if (rsm && (rsm->r_flags & BBR_ACKED)) {
7155 (nrsm->r_flags & BBR_ACKED)) {
7162 (nrsm->r_flags & BBR_ACKED)) {
7181 bbr->r_ctl.rc_sacklast = TAILQ_NEXT(rsm, r_next);
7183 bbr->r_ctl.rc_sacklast = NULL;
7195 while (rsm && (rsm->r_flags & BBR_ACKED)) {
7198 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
7200 if (rsm->r_in_tmap) {
7202 bbr, rsm, rsm->r_flags);
7205 oflags = rsm->r_flags;
7206 if (rsm->r_flags & BBR_MARKED_LOST) {
7207 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7208 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7209 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7211 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7213 rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS | BBR_MARKED_LOST);
7214 rsm->r_flags |= BBR_WAS_RENEGED;
7215 rsm->r_flags |= BBR_RXT_CLEARED;
7216 bbr_log_type_rsmclear(bbr, bbr->r_ctl.rc_rcvtime, rsm, oflags, __LINE__);
7219 TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7222 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, tmap, rsm, r_tnext);
7225 tmap->r_in_tmap = 1;
7240 sack_filter_clear(&bbr->r_ctl.bbr_sf, th_ack);
7250 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
7251 cts = bbr->r_ctl.rc_rcvtime;
7252 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7253 if (rsm && (rsm->r_flags & BBR_HAS_SYN)) {
7254 if ((rsm->r_end - rsm->r_start) <= 1) {
7256 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
7257 rsm->r_rtr_bytes = 0;
7258 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);
7259 if (rsm->r_in_tmap) {
7260 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7261 rsm->r_in_tmap = 0;
7263 if (bbr->r_ctl.rc_next == rsm) {
7265 bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7272 rsm->r_flags &= ~BBR_HAS_SYN;
7273 rsm->r_start++;
7301 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
7302 cts = bbr->r_ctl.rc_rcvtime;
7304 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7306 maxseg = tp->t_maxseg - bbr->rc_last_options;
7307 p_maxseg = min(bbr->r_ctl.rc_pace_max_segs, maxseg);
7308 th_ack = th->th_ack;
7309 if (SEQ_GT(th_ack, tp->snd_una)) {
7311 bbr->rc_tp->t_acktime = ticks;
7313 if (SEQ_LEQ(th_ack, tp->snd_una)) {
7317 if (rsm && SEQ_GT(th_ack, rsm->r_start)) {
7318 changed = th_ack - rsm->r_start;
7319 } else if ((rsm == NULL) && ((th_ack - 1) == tp->iss)) {
7325 if ((to->to_flags & TOF_TS) && (to->to_tsecr != 0)) {
7332 ts = bbr_ts_convert(to->to_tsecr);
7333 now = bbr_ts_convert(tcp_tv_to_mssectick(&bbr->rc_tv));
7334 rtt = now - ts;
7338 tp->iss, 0, cts,
7339 BBR_RTT_BY_TIMESTAMP, tp->iss, 0);
7340 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
7342 bbr->r_wanted_output = 1;
7355 bbr->r_wanted_output = 1;
7358 if (tp->t_flags & TF_SENTFIN) {
7365 th, tp->t_state, bbr,
7366 tp->snd_una, tp->snd_max, changed);
7371 if (SEQ_LT(th_ack, rsm->r_start)) {
7375 rsm->r_start,
7376 th_ack, tp->t_state,
7377 bbr->r_state, bbr);
7378 panic("th-ack is bad bbr:%p tp:%p", bbr, tp);
7381 } else if (th_ack == rsm->r_start) {
7390 rsm->r_dupack = 0;
7392 if (SEQ_GEQ(th_ack, rsm->r_end)) {
7396 if (rsm->r_flags & BBR_ACKED) {
7398 * It was acked on the scoreboard -- remove it from
7401 p_acked += (rsm->r_end - rsm->r_start);
7402 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
7403 if (bbr->r_ctl.rc_sacked == 0)
7404 bbr->r_ctl.rc_sacklast = NULL;
7407 if (rsm->r_flags & BBR_MARKED_LOST) {
7408 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7410 if (rsm->r_flags & BBR_SACK_PASSED) {
7417 bbr->r_ctl.rc_reorder_ts = cts;
7418 if (rsm->r_flags & BBR_MARKED_LOST) {
7419 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7420 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7422 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7425 rsm->r_flags &= ~BBR_MARKED_LOST;
7427 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
7428 rsm->r_rtr_bytes = 0;
7429 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);
7430 if (rsm->r_in_tmap) {
7431 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7432 rsm->r_in_tmap = 0;
7434 if (bbr->r_ctl.rc_next == rsm) {
7436 bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7440 left = th_ack - rsm->r_end;
7444 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7449 if (rsm->r_flags & BBR_ACKED) {
7451 * It was acked on the scoreboard -- remove it from total
7452 * for the part being cum-acked.
7454 p_acked += (rsm->r_end - rsm->r_start);
7455 bbr->r_ctl.rc_sacked -= (th_ack - rsm->r_start);
7456 if (bbr->r_ctl.rc_sacked == 0)
7457 bbr->r_ctl.rc_sacklast = NULL;
7468 if ((rsm->r_flags & BBR_MARKED_LOST) &&
7469 ((rsm->r_flags & BBR_ACKED) == 0)) {
7475 bbr->r_ctl.rc_lost_bytes -= th_ack - rsm->r_start;
7478 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
7479 rsm->r_rtr_bytes = 0;
7481 rsm->r_start = th_ack;
7484 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7485 if (rsm && (rsm->r_flags & BBR_ACKED) && (th_ack == rsm->r_start)) {
7491 * us snd_una up to (rsm->r_end). We need to undo the acked
7495 * rsm->r_start in case we get an old ack where th_ack is
7498 bbr_peer_reneges(bbr, rsm, th->th_ack);
7500 if ((to->to_flags & TOF_SACK) == 0) {
7504 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);
7506 last_seq = rsm->r_end;
7508 last_seq = tp->snd_max;
7511 if (SEQ_GT(th_ack, tp->snd_una))
7514 ack_point = tp->snd_una;
7515 for (i = 0; i < to->to_nsacks; i++) {
7516 bcopy((to->to_sacks + i * TCPOLEN_SACK),
7522 SEQ_LT(sack.start, tp->snd_max) &&
7524 SEQ_LEQ(sack.end, tp->snd_max)) {
7525 if ((bbr->r_ctl.rc_num_small_maps_alloced > bbr_sack_block_limit) &&
7527 ((sack.end - sack.start) < (p_maxseg / 8))) {
7540 * Its a D-SACK block.
7551 new_sb = sack_filter_blks(tp, &bbr->r_ctl.bbr_sf, sack_blocks,
7552 num_sack_blks, th->th_ack);
7553 ctf_log_sack_filter(bbr->rc_tp, new_sb, sack_blocks);
7555 BBR_STAT_ADD(bbr_sack_blocks_skip, (num_sack_blks - new_sb));
7593 * Now collapse out the dup-sack and
7601 num_sack_blks--;
7608 rsm = bbr->r_ctl.rc_sacklast;
7612 bbr->r_wanted_output = 1;
7619 if ((sack_changed) && (!IN_RECOVERY(tp->t_flags))) {
7630 bbr->r_wanted_output = 1;
7635 if (bbr->r_ctl.rc_resend == NULL) {
7636 bbr->r_ctl.rc_resend = rsm;
7640 if (IN_RECOVERY(tp->t_flags) && (entered_recovery == 0)) {
7642 * See if we need to rack-retransmit anything if so set it
7646 if (bbr->r_ctl.rc_resend == NULL) {
7647 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);
7652 * ack-received code to augment what was changed between th_ack <->
7663 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
7664 if (rsm && (rsm->r_dupack < 0xff)) {
7665 rsm->r_dupack++;
7666 if (rsm->r_dupack >= DUP_ACK_THRESHOLD)
7667 bbr->r_wanted_output = 1;
7674 * For ret_val if its 0 the TCB is locked and valid, if its non-zero
7692 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
7693 lost = bbr->r_ctl.rc_lost;
7694 nsegs = max(1, m->m_pkthdr.lro_nsegs);
7695 if (SEQ_GEQ(tp->snd_una, tp->iss + (65535 << tp->snd_scale))) {
7697 tp->t_flags2 |= TF2_NO_ISS_CHECK;
7703 if (tp->t_flags2 & TF2_NO_ISS_CHECK) {
7705 seq_min = tp->snd_una - tp->max_sndwnd;
7708 if (SEQ_GT(tp->iss + 1, tp->snd_una - tp->max_sndwnd)) {
7710 seq_min = tp->iss + 1;
7717 seq_min = tp->snd_una - tp->max_sndwnd;
7721 if (SEQ_LT(th->th_ack, seq_min)) {
7728 bbr->r_wanted_output = 1;
7732 if (SEQ_GT(th->th_ack, tp->snd_max)) {
7734 bbr->r_wanted_output = 1;
7737 if (SEQ_GEQ(th->th_ack, tp->snd_una) || to->to_nsacks) {
7739 if (bbr->rc_in_persist)
7740 tp->t_rxtshift = 0;
7741 if ((th->th_ack == tp->snd_una) && (tiwin == tp->snd_wnd))
7745 bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, (bbr->r_ctl.rc_lost > lost));
7746 if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) {
7751 if (th->th_ack == tp->snd_una) {
7753 if (bbr->r_state == TCPS_SYN_SENT) {
7756 * the SYN-ACK is processed in syn_sent
7773 if (tp->t_flags & TF_NEEDSYN) {
7775 * T/TCP: Connection was half-synchronized, and our SYN has
7777 * to non-starred state, increment snd_una for ACK of SYN,
7780 tp->t_flags &= ~TF_NEEDSYN;
7781 tp->snd_una++;
7783 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
7785 tp->rcv_scale = tp->request_r_scale;
7801 if (tp->t_flags & TF_PREVVALID) {
7802 tp->t_flags &= ~TF_PREVVALID;
7803 if (tp->t_rxtshift == 1 &&
7804 (int)(ticks - tp->t_badrxtwin) < 0)
7808 acked_amount = min(acked, (int)sbavail(&so->so_snd));
7809 tp->snd_wnd -= acked_amount;
7810 mfree = sbcut_locked(&so->so_snd, acked_amount);
7814 if (SEQ_GT(th->th_ack, tp->snd_una)) {
7817 tp->snd_una = th->th_ack;
7818 bbr_ack_received(tp, bbr, th, acked, sack_changed, prev_acked, __LINE__, (bbr->r_ctl.rc_lost - lost));
7819 if (IN_RECOVERY(tp->t_flags)) {
7820 if (SEQ_LT(th->th_ack, tp->snd_recover) &&
7821 (SEQ_LT(th->th_ack, tp->snd_max))) {
7827 if (SEQ_GT(tp->snd_una, tp->snd_recover)) {
7828 tp->snd_recover = tp->snd_una;
7830 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
7831 tp->snd_nxt = tp->snd_max;
7833 if (tp->snd_una == tp->snd_max) {
7837 if (sbavail(&so->so_snd) == 0)
7838 bbr->rc_tp->t_acktime = 0;
7839 if ((sbused(&so->so_snd) == 0) &&
7840 (tp->t_flags & TF_SENTFIN)) {
7843 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
7844 if (bbr->rc_in_persist == 0) {
7845 bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime;
7847 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
7848 bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime);
7854 if ((tp->t_state >= TCPS_FIN_WAIT_1) &&
7855 (sbavail(&so->so_snd) == 0) &&
7856 (tp->t_flags2 & TF2_DROP_AF_DATA)) {
7863 /* tcp_close will kill the inp pre-log the Reset */
7871 bbr->r_wanted_output = 1;
7881 if (bbr->rc_in_persist == 0) {
7883 bbr->r_ctl.rc_last_delay_val = 0;
7884 tp->t_rxtshift = 0;
7885 bbr->rc_in_persist = 1;
7886 bbr->r_ctl.rc_went_idle_time = cts;
7890 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
7893 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
7894 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {
7900 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
7903 bbr->r_ctl.rc_bbr_state_time = cts;
7916 if (bbr->rc_use_idle_restart) {
7917 bbr->rc_bbr_state = BBR_STATE_IDLE_EXIT;
7924 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
7925 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
7928 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;
7929 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;
7931 } else if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {
7942 if (bbr->rc_in_persist == 0)
7944 idle_time = bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time);
7945 bbr->rc_in_persist = 0;
7946 bbr->rc_hit_state_1 = 0;
7947 bbr->r_ctl.rc_del_time = cts;
7953 if (tcp_in_hpts(bbr->rc_tp)) {
7954 tcp_hpts_remove(bbr->rc_tp);
7955 bbr->rc_timer_first = 0;
7956 bbr->r_ctl.rc_hpts_flags = 0;
7957 bbr->r_ctl.rc_last_delay_val = 0;
7958 bbr->r_ctl.rc_hptsi_agg_delay = 0;
7959 bbr->r_agg_early_set = 0;
7960 bbr->r_ctl.rc_agg_early = 0;
7970 bbr->r_ctl.last_in_probertt = bbr->r_ctl.rc_rtt_shrinks = cts;
7972 tp->t_rxtshift = 0;
7983 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
7984 /* Time un-freezes for the state */
7985 bbr->r_ctl.rc_bbr_state_time = cts;
7986 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) ||
7987 (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT)) {
7989 * If we are going back to probe-bw
8022 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
8023 max_seq = bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd;
8025 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
8027 if (rsm->r_flags & BBR_RWND_COLLAPSED)
8028 rsm->r_flags &= ~BBR_RWND_COLLAPSED;
8029 if (SEQ_GEQ(max_seq, rsm->r_start) &&
8030 SEQ_GEQ(rsm->r_end, max_seq)) {
8035 bbr->rc_has_collapsed = 0;
8048 if ((max_seq != rsm->r_start) &&
8049 (max_seq != rsm->r_end)){
8053 res1 = max_seq - rsm->r_start;
8054 res2 = rsm->r_end - max_seq;
8059 } else if (bbr->r_ctl.rc_num_small_maps_alloced < bbr_sack_block_limit) {
8065 if (max_seq == rsm->r_start) {
8068 } else if (max_seq == rsm->r_end) {
8075 } else if (can_split && SEQ_LT(max_seq, rsm->r_end)) {
8086 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
8087 if (rsm->r_in_tmap) {
8088 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
8089 nrsm->r_in_tmap = 1;
8102 TAILQ_FOREACH_FROM(nrsm, &bbr->r_ctl.rc_map, r_next) {
8103 nrsm->r_flags |= BBR_RWND_COLLAPSED;
8105 bbr->rc_has_collapsed = 1;
8116 TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
8117 if (rsm->r_flags & BBR_RWND_COLLAPSED) {
8119 rsm->r_flags &= ~BBR_RWND_COLLAPSED;
8125 (bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd), 0, cleared);
8126 bbr->rc_has_collapsed = 0;
8147 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8149 nsegs = max(1, m->m_pkthdr.lro_nsegs);
8151 (SEQ_LT(tp->snd_wl1, th->th_seq) ||
8152 (tp->snd_wl1 == th->th_seq && (SEQ_LT(tp->snd_wl2, th->th_ack) ||
8153 (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd))))) {
8156 tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd)
8158 tp->snd_wnd = tiwin;
8159 tp->snd_wl1 = th->th_seq;
8160 tp->snd_wl2 = th->th_ack;
8161 if (tp->snd_wnd > tp->max_sndwnd)
8162 tp->max_sndwnd = tp->snd_wnd;
8163 bbr->r_wanted_output = 1;
8165 if ((tp->snd_wl2 == th->th_ack) && (tiwin < tp->snd_wnd)) {
8166 tp->snd_wnd = tiwin;
8167 tp->snd_wl1 = th->th_seq;
8168 tp->snd_wl2 = th->th_ack;
8171 if (tp->snd_wnd < ctf_outstanding(tp))
8174 else if (bbr->rc_has_collapsed)
8177 if ((bbr->rc_in_persist != 0) &&
8178 (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2),
8181 * Make the rate persist at end of persist mode if idle long
8184 bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8187 bbr->r_wanted_output = 1;
8190 if ((bbr->rc_in_persist == 0) &&
8191 (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
8192 TCPS_HAVEESTABLISHED(tp->t_state) &&
8193 (tp->snd_max == tp->snd_una) &&
8194 sbavail(&so->so_snd) &&
8195 (sbavail(&so->so_snd) > tp->snd_wnd)) {
8197 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8199 if (tp->t_flags2 & TF2_DROP_AF_DATA) {
8209 tp->rcv_up = tp->rcv_nxt;
8214 * This process logically involves adjusting tp->rcv_wnd as data is
8219 tfo_syn = ((tp->t_state == TCPS_SYN_RECEIVED) &&
8220 (tp->t_flags & TF_FASTOPEN));
8222 TCPS_HAVERCVDFIN(tp->t_state) == 0) {
8223 tcp_seq save_start = th->th_seq;
8224 tcp_seq save_rnxt = tp->rcv_nxt;
8239 if (th->th_seq == tp->rcv_nxt &&
8241 (TCPS_HAVEESTABLISHED(tp->t_state) ||
8246 if (so->so_rcv.sb_shlim) {
8249 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt,
8259 bbr->bbr_segs_rcvd += max(1, nsegs);
8260 tp->t_flags |= TF_DELACK;
8261 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8263 bbr->r_wanted_output = 1;
8264 tp->t_flags |= TF_ACKNOW;
8266 tp->rcv_nxt += tlen;
8268 ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) &&
8269 (tp->t_fbyte_in == 0)) {
8270 tp->t_fbyte_in = ticks;
8271 if (tp->t_fbyte_in == 0)
8272 tp->t_fbyte_in = 1;
8273 if (tp->t_fbyte_out && tp->t_fbyte_in)
8274 tp->t_flags2 |= TF2_FBYTES_COMPLETE;
8280 if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
8286 sbappendstream_locked(&so->so_rcv, m, 0);
8290 if (so->so_rcv.sb_shlim && appended != mcnt)
8291 counter_fo_release(so->so_rcv.sb_shlim,
8292 mcnt - appended);
8305 tp->t_flags |= TF_ACKNOW;
8306 if (tp->t_flags & TF_WAKESOR) {
8307 tp->t_flags &= ~TF_WAKESOR;
8312 if ((tp->t_flags & TF_SACK_PERMIT) &&
8314 TCPS_HAVEESTABLISHED(tp->t_state)) {
8322 } else if ((tlen > 0) && SEQ_GT(tp->rcv_nxt, save_rnxt)) {
8323 if ((tp->rcv_numsacks >= 1) &&
8324 (tp->sackblks[0].end == save_start)) {
8330 tp->sackblks[0].start,
8331 tp->sackblks[0].end);
8355 if (TCPS_HAVERCVDFIN(tp->t_state) == 0) {
8359 * If connection is half-synchronized (ie NEEDSYN
8365 if (tp->t_flags & TF_NEEDSYN) {
8366 tp->t_flags |= TF_DELACK;
8368 __LINE__, bbr->r_ctl.rc_rcvtime);
8370 tp->t_flags |= TF_ACKNOW;
8372 tp->rcv_nxt++;
8374 switch (tp->t_state) {
8380 tp->t_starttime = ticks;
8396 * starting the time-wait timer, turning off the
8400 bbr->rc_timer_first = 1;
8402 __LINE__, bbr->r_ctl.rc_rcvtime);
8410 if ((tp->t_flags & TF_ACKNOW) ||
8411 (sbavail(&so->so_snd) > ctf_outstanding(tp))) {
8412 bbr->r_wanted_output = 1;
8419 * have broken out the fast-data path also just like
8420 * the fast-ack. Return 1 if we processed the packet
8421 * return 0 if you need to take the "slow-path".
8436 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8443 if (bbr->r_ctl.rc_resend != NULL) {
8446 if (tiwin && tiwin != tp->snd_wnd) {
8449 if (__predict_false((tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN)))) {
8452 if (__predict_false((to->to_flags & TOF_TS) &&
8453 (TSTMP_LT(to->to_tsval, tp->ts_recent)))) {
8456 if (__predict_false((th->th_ack != tp->snd_una))) {
8459 if (__predict_false(tlen > sbspace(&so->so_rcv))) {
8462 if ((to->to_flags & TOF_TS) != 0 &&
8463 SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {
8464 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
8465 tp->ts_recent = to->to_tsval;
8468 * This is a pure, in-sequence data packet with nothing on the
8471 nsegs = max(1, m->m_pkthdr.lro_nsegs);
8474 if (so->so_rcv.sb_shlim) {
8477 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt,
8486 if (tp->rcv_numsacks)
8489 tp->rcv_nxt += tlen;
8491 ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) &&
8492 (tp->t_fbyte_in == 0)) {
8493 tp->t_fbyte_in = ticks;
8494 if (tp->t_fbyte_in == 0)
8495 tp->t_fbyte_in = 1;
8496 if (tp->t_fbyte_out && tp->t_fbyte_in)
8497 tp->t_flags2 |= TF2_FBYTES_COMPLETE;
8502 tp->snd_wl1 = th->th_seq;
8506 tp->rcv_up = tp->rcv_nxt;
8513 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
8522 so->so_rcv.sb_flags &= ~SB_AUTOSIZE;
8528 sbappendstream_locked(&so->so_rcv, m, 0);
8534 if (so->so_rcv.sb_shlim && mcnt != appended)
8535 counter_fo_release(so->so_rcv.sb_shlim, mcnt - appended);
8538 bbr->bbr_segs_rcvd += max(1, nsegs);
8539 tp->t_flags |= TF_DELACK;
8540 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8542 bbr->r_wanted_output = 1;
8543 tp->t_flags |= TF_ACKNOW;
8551 * in sequence to remain in the fast-path. We also add
8555 * slow-path. If we return 1, then all is well and
8569 if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) {
8573 if (__predict_false(SEQ_GT(th->th_ack, tp->snd_max))) {
8581 if (__predict_false(tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN))) {
8585 if ((to->to_flags & TOF_TS) && __predict_false(TSTMP_LT(to->to_tsval, tp->ts_recent))) {
8589 if (__predict_false(IN_RECOVERY(tp->t_flags))) {
8593 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8594 if (__predict_false(bbr->r_ctl.rc_resend != NULL)) {
8598 if (__predict_false(bbr->rc_in_persist != 0)) {
8599 /* In persist mode */
8602 if (bbr->r_ctl.rc_sacked) {
8606 /* Ok if we reach here, we can process a fast-ack */
8607 nsegs = max(1, m->m_pkthdr.lro_nsegs);
8614 bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, 0);
8616 if (tiwin != tp->snd_wnd) {
8617 tp->snd_wnd = tiwin;
8618 tp->snd_wl1 = th->th_seq;
8619 if (tp->snd_wnd > tp->max_sndwnd)
8620 tp->max_sndwnd = tp->snd_wnd;
8623 if ((bbr->rc_in_persist != 0) &&
8624 (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2),
8626 bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8627 bbr->r_wanted_output = 1;
8630 if ((bbr->rc_in_persist == 0) &&
8631 (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
8632 TCPS_HAVEESTABLISHED(tp->t_state) &&
8633 (tp->snd_max == tp->snd_una) &&
8634 sbavail(&so->so_snd) &&
8635 (sbavail(&so->so_snd) > tp->snd_wnd)) {
8637 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8644 if ((to->to_flags & TOF_TS) != 0 &&
8645 SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {
8646 tp->ts_recent_age = bbr->r_ctl.rc_rcvtime;
8647 tp->ts_recent = to->to_tsval;
8657 if (tp->t_flags & TF_PREVVALID) {
8658 tp->t_flags &= ~TF_PREVVALID;
8659 if (tp->t_rxtshift == 1 &&
8660 (int)(ticks - tp->t_badrxtwin) < 0)
8679 sbdrop(&so->so_snd, acked);
8681 if (SEQ_GT(th->th_ack, tp->snd_una))
8683 tp->snd_una = th->th_ack;
8684 if (tp->snd_wnd < ctf_outstanding(tp))
8687 else if (bbr->rc_has_collapsed)
8690 if (SEQ_GT(tp->snd_una, tp->snd_recover)) {
8691 tp->snd_recover = tp->snd_una;
8697 tp->snd_wl2 = th->th_ack;
8701 * otherwise restart timer using current (possibly backed-off)
8708 if (tp->snd_una == tp->snd_max) {
8711 if (sbavail(&so->so_snd) == 0)
8712 bbr->rc_tp->t_acktime = 0;
8713 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8714 if (bbr->rc_in_persist == 0) {
8715 bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime;
8717 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
8718 bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime);
8724 bbr->r_wanted_output = 1;
8726 if (sbavail(&so->so_snd)) {
8727 bbr->r_wanted_output = 1;
8749 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8755 * this is an acceptable SYN segment initialize tp->rcv_nxt and
8756 * tp->irs if seg contains ack then advance tp->snd_una. BRR does
8763 (SEQ_LEQ(th->th_ack, tp->iss) ||
8764 SEQ_GT(th->th_ack, tp->snd_max))) {
8784 tp->irs = th->th_seq;
8795 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
8797 tp->rcv_scale = tp->request_r_scale;
8799 tp->rcv_adv += min(tp->rcv_wnd,
8800 TCP_MAXWIN << tp->rcv_scale);
8805 if ((tp->t_flags & TF_FASTOPEN) &&
8806 (tp->snd_una != tp->snd_max)) {
8807 tp->snd_nxt = th->th_ack;
8815 bbr->bbr_segs_rcvd += 1;
8816 tp->t_flags |= TF_DELACK;
8817 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8819 bbr->r_wanted_output = 1;
8820 tp->t_flags |= TF_ACKNOW;
8822 if (SEQ_GT(th->th_ack, tp->iss)) {
8829 if (SEQ_GT(th->th_ack, tp->snd_una)) {
8835 * ack-processing since the
8836 * data stream in our send-map
8842 tp->snd_una++;
8846 * SYN_SENT --> ESTABLISHED SYN_SENT* --> FIN_WAIT_1
8848 tp->t_starttime = ticks;
8849 if (tp->t_flags & TF_NEEDFIN) {
8851 tp->t_flags &= ~TF_NEEDFIN;
8861 * Received initial SYN in SYN-SENT[*] state => simultaneous
8864 * half-synchronized. Otherwise, do 3-way handshake:
8865 * SYN-SENT -> SYN-RECEIVED SYN-SENT* -> SYN-RECEIVED* If
8868 tp->t_flags |= (TF_ACKNOW | TF_NEEDSYN | TF_SONOTCONN);
8872 * Advance th->th_seq to correspond to first data byte. If data,
8875 th->th_seq++;
8876 if (tlen > tp->rcv_wnd) {
8877 todrop = tlen - tp->rcv_wnd;
8878 m_adj(m, -todrop);
8879 tlen = tp->rcv_wnd;
8884 tp->snd_wl1 = th->th_seq - 1;
8885 tp->rcv_up = th->th_seq;
8893 if ((to->to_flags & TOF_TS) != 0) {
8896 t = tcp_tv_to_mssectick(&bbr->rc_tv);
8897 if (TSTMP_GEQ(t, to->to_tsecr)) {
8898 rtt = t - to->to_tsecr;
8904 apply_filter_min_small(&bbr->r_ctl.rc_rttprop,
8905 rtt, bbr->r_ctl.rc_rcvtime);
8911 if (tp->t_state == TCPS_FIN_WAIT_1) {
8928 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
8959 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8962 (tp->t_fin_is_rst && (thflags & TH_FIN)))
8965 (SEQ_LEQ(th->th_ack, tp->snd_una) ||
8966 SEQ_GT(th->th_ack, tp->snd_max))) {
8971 if (tp->t_flags & TF_FASTOPEN) {
8983 /* non-initial SYN is ignored */
8984 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RXT) ||
8985 (bbr->r_ctl.rc_hpts_flags & PACE_TMR_TLP) ||
8986 (bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK)) {
8999 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9000 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9005 * In the SYN-RECEIVED state, validate that the packet belongs to
9011 if (SEQ_LT(th->th_seq, tp->irs)) {
9033 if ((to->to_flags & TOF_TS) != 0 &&
9034 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9035 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9037 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9038 tp->ts_recent = to->to_tsval;
9040 tp->snd_wnd = tiwin;
9042 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9043 * is on (half-synchronized state), then queue data for later
9047 if (tp->t_flags & TF_FASTOPEN) {
9054 if (tp->t_flags & TF_SONOTCONN) {
9055 tp->t_flags &= ~TF_SONOTCONN;
9059 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
9061 tp->rcv_scale = tp->request_r_scale;
9067 if ((to->to_flags & TOF_TS) != 0) {
9070 t = tcp_tv_to_mssectick(&bbr->rc_tv);
9071 if (TSTMP_GEQ(t, to->to_tsecr)) {
9072 rtt = t - to->to_tsecr;
9078 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, bbr->r_ctl.rc_rcvtime);
9084 if ((tp->t_flags & TF_FASTOPEN) && tp->t_tfo_pending) {
9085 tcp_fastopen_decrement_counter(tp->t_tfo_pending);
9086 tp->t_tfo_pending = NULL;
9089 * Make transitions: SYN-RECEIVED -> ESTABLISHED SYN-RECEIVED* ->
9090 * FIN-WAIT-1
9092 tp->t_starttime = ticks;
9093 if (tp->t_flags & TF_NEEDFIN) {
9095 tp->t_flags &= ~TF_NEEDFIN;
9106 if (!(tp->t_flags & TF_FASTOPEN))
9114 if (SEQ_GT(th->th_ack, tp->snd_una) && !(tp->t_flags & TF_NEEDSYN))
9115 tp->snd_una++;
9123 if (tp->t_flags & TF_WAKESOR) {
9124 tp->t_flags &= ~TF_WAKESOR;
9129 tp->snd_wl1 = th->th_seq - 1;
9133 if (tp->t_state == TCPS_FIN_WAIT_1) {
9150 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
9181 * uni-directional data xfer. If the packet has no control flags,
9182 * is in-sequence, the window didn't change and we're not
9186 * waiting for space. If the length is non-zero and the ack didn't
9187 * move, we're the receiver side. If we're getting packets in-order
9190 * hidden state-flags are also off. Since we check for
9193 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9194 if (bbr->r_ctl.rc_delivered < (4 * tp->t_maxseg)) {
9200 bbr->r_ctl.rc_init_rwnd = max(tiwin, tp->snd_wnd);
9202 if (__predict_true(((to->to_flags & TOF_SACK) == 0)) &&
9205 __predict_true(th->th_seq == tp->rcv_nxt)) {
9221 (tp->t_fin_is_rst && (thflags & TH_FIN)))
9235 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9236 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9257 if ((to->to_flags & TOF_TS) != 0 &&
9258 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9259 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9261 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9262 tp->ts_recent = to->to_tsval;
9265 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9266 * is on (half-synchronized state), then queue data for later
9270 if (tp->t_flags & TF_NEEDSYN) {
9273 } else if (tp->t_flags & TF_ACKNOW) {
9275 bbr->r_wanted_output = 1;
9288 if (sbavail(&so->so_snd)) {
9315 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9318 (tp->t_fin_is_rst && (thflags & TH_FIN)))
9332 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9333 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9354 if ((to->to_flags & TOF_TS) != 0 &&
9355 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9356 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9358 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9359 tp->ts_recent = to->to_tsval;
9362 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9363 * is on (half-synchronized state), then queue data for later
9367 if (tp->t_flags & TF_NEEDSYN) {
9370 } else if (tp->t_flags & TF_ACKNOW) {
9372 bbr->r_wanted_output = 1;
9385 if (sbavail(&so->so_snd)) {
9401 if (bbr->rc_allow_data_af_clo == 0) {
9404 /* tcp_close will kill the inp pre-log the Reset */
9411 if (sbavail(&so->so_snd) == 0)
9414 tp->rcv_nxt = th->th_seq + *tlen;
9415 tp->t_flags2 |= TF2_DROP_AF_DATA;
9416 bbr->r_wanted_output = 1;
9437 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9440 (tp->t_fin_is_rst && (thflags & TH_FIN)))
9454 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9455 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9468 if ((tp->t_flags & TF_CLOSED) && tlen &&
9485 if ((to->to_flags & TOF_TS) != 0 &&
9486 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9487 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9489 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9490 tp->ts_recent = to->to_tsval;
9493 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9494 * is on (half-synchronized state), then queue data for later
9498 if (tp->t_flags & TF_NEEDSYN) {
9501 } else if (tp->t_flags & TF_ACKNOW) {
9503 bbr->r_wanted_output = 1;
9526 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
9535 if (sbavail(&so->so_snd)) {
9562 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9565 (tp->t_fin_is_rst && (thflags & TH_FIN)))
9579 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9580 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9601 if ((to->to_flags & TOF_TS) != 0 &&
9602 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9603 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9605 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9606 tp->ts_recent = to->to_tsval;
9609 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9610 * is on (half-synchronized state), then queue data for later
9614 if (tp->t_flags & TF_NEEDSYN) {
9617 } else if (tp->t_flags & TF_ACKNOW) {
9619 bbr->r_wanted_output = 1;
9637 if (sbavail(&so->so_snd)) {
9664 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9667 (tp->t_fin_is_rst && (thflags & TH_FIN)))
9681 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9682 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9703 if ((to->to_flags & TOF_TS) != 0 &&
9704 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9705 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9707 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9708 tp->ts_recent = to->to_tsval;
9711 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9712 * is on (half-synchronized state), then queue data for later
9716 if (tp->t_flags & TF_NEEDSYN) {
9719 } else if (tp->t_flags & TF_ACKNOW) {
9721 bbr->r_wanted_output = 1;
9739 if (sbavail(&so->so_snd)) {
9766 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9768 /* Reset receive buffer auto scaling when not in bulk receive mode. */
9770 (tp->t_fin_is_rst && (thflags & TH_FIN)))
9785 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9786 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9800 if ((tp->t_flags & TF_CLOSED) && tlen &&
9817 if ((to->to_flags & TOF_TS) != 0 &&
9818 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9819 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9821 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9822 tp->ts_recent = to->to_tsval;
9825 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9826 * is on (half-synchronized state), then queue data for later
9830 if (tp->t_flags & TF_NEEDSYN) {
9833 } else if (tp->t_flags & TF_ACKNOW) {
9835 bbr->r_wanted_output = 1;
9848 if (sbavail(&so->so_snd)) {
9867 bbr->rc_in_persist = 1;
9869 if (tcp_in_hpts(bbr->rc_tp)) {
9870 tcp_hpts_remove(bbr->rc_tp);
9877 bbr->rc_use_google = 1;
9878 bbr->rc_no_pacing = 0;
9879 bbr->r_ctl.bbr_google_discount = bbr_google_discount;
9880 bbr->r_use_policer = bbr_policer_detection_enabled;
9881 bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10);
9882 bbr->bbr_use_rack_cheat = 0;
9883 bbr->r_ctl.rc_incr_tmrs = 0;
9884 bbr->r_ctl.rc_inc_tcp_oh = 0;
9885 bbr->r_ctl.rc_inc_ip_oh = 0;
9886 bbr->r_ctl.rc_inc_enet_oh = 0;
9887 reset_time(&bbr->r_ctl.rc_delrate,
9889 reset_time_small(&bbr->r_ctl.rc_rttprop,
9891 tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv));
9897 bbr->rc_use_google = 0;
9898 bbr->r_ctl.bbr_google_discount = 0;
9899 bbr->no_pacing_until = bbr_no_pacing_until;
9900 bbr->r_use_policer = 0;
9901 if (bbr->no_pacing_until)
9902 bbr->rc_no_pacing = 1;
9904 bbr->rc_no_pacing = 0;
9906 bbr->bbr_use_rack_cheat = 1;
9908 bbr->bbr_use_rack_cheat = 0;
9910 bbr->r_ctl.rc_incr_tmrs = 1;
9912 bbr->r_ctl.rc_incr_tmrs = 0;
9914 bbr->r_ctl.rc_inc_tcp_oh = 1;
9916 bbr->r_ctl.rc_inc_tcp_oh = 0;
9918 bbr->r_ctl.rc_inc_ip_oh = 1;
9920 bbr->r_ctl.rc_inc_ip_oh = 0;
9922 bbr->r_ctl.rc_inc_enet_oh = 1;
9924 bbr->r_ctl.rc_inc_enet_oh = 0;
9925 bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;
9926 reset_time(&bbr->r_ctl.rc_delrate,
9928 reset_time_small(&bbr->r_ctl.rc_rttprop,
9930 tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv));
9933 * Return 0 on success, non-zero on failure
9956 bbr->rtt_valid = 0;
9957 tp->t_flags2 |= TF2_CANNOT_DO_ECN;
9958 tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
9960 tp->t_flags2 &= ~TF2_MBUF_QUEUE_READY;
9961 tp->t_flags2 &= ~TF2_DONT_SACK_QUEUE;
9962 tp->t_flags2 &= ~TF2_MBUF_ACKCMP;
9963 tp->t_flags2 &= ~TF2_MBUF_L_ACKS;
9965 TAILQ_INIT(&bbr->r_ctl.rc_map);
9966 TAILQ_INIT(&bbr->r_ctl.rc_free);
9967 TAILQ_INIT(&bbr->r_ctl.rc_tmap);
9968 bbr->rc_tp = tp;
9969 bbr->rc_inp = inp;
9970 cts = tcp_get_usecs(&bbr->rc_tv);
9971 tp->t_acktime = 0;
9972 bbr->rc_allow_data_af_clo = bbr_ignore_data_after_close;
9973 bbr->r_ctl.rc_reorder_fade = bbr_reorder_fade;
9974 bbr->rc_tlp_threshold = bbr_tlp_thresh;
9975 bbr->r_ctl.rc_reorder_shift = bbr_reorder_thresh;
9976 bbr->r_ctl.rc_pkt_delay = bbr_pkt_delay;
9977 bbr->r_ctl.rc_min_to = bbr_min_to;
9978 bbr->rc_bbr_state = BBR_STATE_STARTUP;
9979 bbr->r_ctl.bbr_lost_at_state = 0;
9980 bbr->r_ctl.rc_lost_at_startup = 0;
9981 bbr->rc_all_timers_stopped = 0;
9982 bbr->r_ctl.rc_bbr_lastbtlbw = 0;
9983 bbr->r_ctl.rc_pkt_epoch_del = 0;
9984 bbr->r_ctl.rc_pkt_epoch = 0;
9985 bbr->r_ctl.rc_lowest_rtt = 0xffffffff;
9986 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_high_gain;
9987 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain;
9988 bbr->r_ctl.rc_went_idle_time = cts;
9989 bbr->rc_pacer_started = cts;
9990 bbr->r_ctl.rc_pkt_epoch_time = cts;
9991 bbr->r_ctl.rc_rcvtime = cts;
9992 bbr->r_ctl.rc_bbr_state_time = cts;
9993 bbr->r_ctl.rc_del_time = cts;
9994 bbr->r_ctl.rc_tlp_rxt_last_time = cts;
9995 bbr->r_ctl.last_in_probertt = cts;
9996 bbr->skip_gain = 0;
9997 bbr->gain_is_limited = 0;
9998 bbr->no_pacing_until = bbr_no_pacing_until;
9999 if (bbr->no_pacing_until)
10000 bbr->rc_no_pacing = 1;
10002 bbr->rc_no_pacing = 0;
10003 bbr->rc_use_google = 1;
10004 bbr->r_ctl.bbr_google_discount = bbr_google_discount;
10005 bbr->r_use_policer = bbr_policer_detection_enabled;
10007 bbr->rc_use_google = 0;
10008 bbr->r_ctl.bbr_google_discount = 0;
10009 bbr->r_use_policer = 0;
10012 bbr->rc_use_ts_limit = 1;
10014 bbr->rc_use_ts_limit = 0;
10016 bbr->ts_can_raise = 1;
10018 bbr->ts_can_raise = 0;
10020 tp->t_delayed_ack = 2;
10022 tp->t_delayed_ack = 0;
10024 tp->t_delayed_ack = V_tcp_delack_enabled;
10026 tp->t_delayed_ack = 2;
10027 if (bbr->rc_use_google == 0)
10028 bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;
10030 bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10);
10031 bbr->r_ctl.rc_min_rto_ms = bbr_rto_min_ms;
10032 bbr->rc_max_rto_sec = bbr_rto_max_sec;
10033 bbr->rc_init_win = bbr_def_init_win;
10034 if (tp->t_flags & TF_REQ_TSTMP)
10035 bbr->rc_last_options = TCP_TS_OVERHEAD;
10036 bbr->r_ctl.rc_pace_max_segs = tp->t_maxseg - bbr->rc_last_options;
10037 bbr->r_ctl.rc_high_rwnd = tp->snd_wnd;
10038 bbr->r_init_rtt = 1;
10042 bbr->bbr_hdw_pace_ena = 1;
10044 bbr->bbr_hdw_pace_ena = 0;
10046 bbr->bbr_init_win_cheat = 1;
10048 bbr->bbr_init_win_cheat = 0;
10049 bbr->r_ctl.bbr_utter_max = bbr_hptsi_utter_max;
10050 bbr->r_ctl.rc_drain_pg = bbr_drain_gain;
10051 bbr->r_ctl.rc_startup_pg = bbr_high_gain;
10052 bbr->rc_loss_exit = bbr_exit_startup_at_loss;
10053 bbr->r_ctl.bbr_rttprobe_gain_val = bbr_rttprobe_gain;
10054 bbr->r_ctl.bbr_hptsi_per_second = bbr_hptsi_per_second;
10055 bbr->r_ctl.bbr_hptsi_segments_delay_tar = bbr_hptsi_segments_delay_tar;
10056 bbr->r_ctl.bbr_hptsi_segments_max = bbr_hptsi_segments_max;
10057 bbr->r_ctl.bbr_hptsi_segments_floor = bbr_hptsi_segments_floor;
10058 bbr->r_ctl.bbr_hptsi_bytes_min = bbr_hptsi_bytes_min;
10059 bbr->r_ctl.bbr_cross_over = bbr_cross_over;
10060 bbr->r_ctl.rc_rtt_shrinks = cts;
10061 if (bbr->rc_use_google) {
10062 setup_time_filter(&bbr->r_ctl.rc_delrate,
10065 setup_time_filter_small(&bbr->r_ctl.rc_rttprop,
10068 setup_time_filter(&bbr->r_ctl.rc_delrate,
10071 setup_time_filter_small(&bbr->r_ctl.rc_rttprop,
10076 bbr->rc_use_idle_restart = 1;
10078 bbr->rc_use_idle_restart = 0;
10079 bbr->r_ctl.rc_bbr_cur_del_rate = 0;
10080 bbr->r_ctl.rc_initial_hptsi_bw = bbr_initial_bw_bps;
10082 bbr->rc_resends_use_tso = 1;
10083 if (tp->snd_una != tp->snd_max) {
10093 rsm->r_rtt_not_allowed = 1;
10094 rsm->r_tim_lastsent[0] = cts;
10095 rsm->r_rtr_cnt = 1;
10096 rsm->r_rtr_bytes = 0;
10097 rsm->r_start = tp->snd_una;
10098 rsm->r_end = tp->snd_max;
10099 rsm->r_dupack = 0;
10100 rsm->r_delivered = bbr->r_ctl.rc_delivered;
10101 rsm->r_ts_valid = 0;
10102 rsm->r_del_ack_ts = tp->ts_recent;
10103 rsm->r_del_time = cts;
10104 if (bbr->r_ctl.r_app_limited_until)
10105 rsm->r_app_limited = 1;
10107 rsm->r_app_limited = 0;
10108 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next);
10109 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
10110 rsm->r_in_tmap = 1;
10111 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)
10112 rsm->r_bbr_state = bbr_state_val(bbr);
10114 rsm->r_bbr_state = 8;
10116 if (bbr_use_rack_resend_cheat && (bbr->rc_use_google == 0))
10117 bbr->bbr_use_rack_cheat = 1;
10118 if (bbr_incr_timers && (bbr->rc_use_google == 0))
10119 bbr->r_ctl.rc_incr_tmrs = 1;
10120 if (bbr_include_tcp_oh && (bbr->rc_use_google == 0))
10121 bbr->r_ctl.rc_inc_tcp_oh = 1;
10122 if (bbr_include_ip_oh && (bbr->rc_use_google == 0))
10123 bbr->r_ctl.rc_inc_ip_oh = 1;
10124 if (bbr_include_enet_oh && (bbr->rc_use_google == 0))
10125 bbr->r_ctl.rc_inc_enet_oh = 1;
10128 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
10129 (tp->t_srtt)) {
10132 rtt = (TICKS_2_USEC(tp->t_srtt) >> TCP_RTT_SHIFT);
10133 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
10151 TCPT_RANGESET(tp->t_rxtcur,
10152 ((tp->t_srtt >> 2) + tp->t_rttvar) >> 1,
10153 tp->t_rttmin, TCPTV_REXMTMAX);
10160 * non-zero if we can't handle the connection. A EAGAIN
10168 if ((tp->t_state == TCPS_CLOSED) ||
10169 (tp->t_state == TCPS_LISTEN)) {
10173 if ((tp->t_state == TCPS_SYN_SENT) ||
10174 (tp->t_state == TCPS_SYN_RECEIVED)) {
10181 if (tp->t_flags & TF_SENTFIN)
10183 if ((tp->t_flags & TF_SACK_PERMIT) || bbr_sack_not_required) {
10196 if (tp->t_fb_ptr) {
10201 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
10202 if (bbr->r_ctl.crte)
10203 tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp);
10205 bbr->rc_tp = NULL;
10206 if (bbr->bbr_hdrw_pacing)
10207 counter_u64_add(bbr_flows_whdwr_pacing, -1);
10209 counter_u64_add(bbr_flows_nohdwr_pacing, -1);
10210 if (bbr->r_ctl.crte != NULL) {
10211 tcp_rel_pacing_rate(bbr->r_ctl.crte, tp);
10212 bbr->r_ctl.crte = NULL;
10214 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
10216 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);
10218 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
10220 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);
10222 TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next);
10224 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);
10226 calc = bbr->r_ctl.rc_high_rwnd - bbr->r_ctl.rc_init_rwnd;
10227 if (calc > (bbr->r_ctl.rc_init_rwnd / 10))
10231 bbr->r_ctl.rc_free_cnt = 0;
10232 uma_zfree(bbr_pcb_zone, tp->t_fb_ptr);
10233 tp->t_fb_ptr = NULL;
10236 tp->snd_nxt = tp->snd_max;
10242 switch (tp->t_state) {
10244 bbr->r_state = TCPS_SYN_SENT;
10245 bbr->r_substate = bbr_do_syn_sent;
10248 bbr->r_state = TCPS_SYN_RECEIVED;
10249 bbr->r_substate = bbr_do_syn_recv;
10252 bbr->r_ctl.rc_init_rwnd = max(win, bbr->rc_tp->snd_wnd);
10253 bbr->r_state = TCPS_ESTABLISHED;
10254 bbr->r_substate = bbr_do_established;
10257 bbr->r_state = TCPS_CLOSE_WAIT;
10258 bbr->r_substate = bbr_do_close_wait;
10261 bbr->r_state = TCPS_FIN_WAIT_1;
10262 bbr->r_substate = bbr_do_fin_wait_1;
10265 bbr->r_state = TCPS_CLOSING;
10266 bbr->r_substate = bbr_do_closing;
10269 bbr->r_state = TCPS_LAST_ACK;
10270 bbr->r_substate = bbr_do_lastack;
10273 bbr->r_state = TCPS_FIN_WAIT_2;
10274 bbr->r_substate = bbr_do_fin_wait_2;
10295 /* Save the lowest srtt we saw in our end of the sub-state */
10296 bbr->rc_hit_state_1 = 0;
10297 if (bbr->r_ctl.bbr_smallest_srtt_this_state != 0xffffffff)
10298 bbr->r_ctl.bbr_smallest_srtt_state2 = bbr->r_ctl.bbr_smallest_srtt_this_state;
10300 bbr->rc_bbr_substate++;
10306 if (bbr->skip_gain) {
10312 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_LEVEL1];
10313 } else if (bbr->gain_is_limited &&
10314 bbr->bbr_hdrw_pacing &&
10315 bbr->r_ctl.crte) {
10325 rate = bbr->r_ctl.crte->rate;
10331 bbr->r_ctl.rc_bbr_hptsi_gain = (uint16_t)gain_calc;
10333 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN];
10336 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN];
10337 if ((bbr->rc_use_google == 0) && (bbr_gain_to_target == 0)) {
10338 bbr->r_ctl.rc_bbr_state_atflight = cts;
10340 bbr->r_ctl.rc_bbr_state_atflight = 0;
10342 bbr->rc_hit_state_1 = 1;
10343 bbr->r_ctl.rc_exta_time_gd = 0;
10344 bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp,
10345 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
10347 bbr->r_ctl.rc_bbr_state_atflight = 0;
10349 bbr->r_ctl.rc_bbr_state_atflight = cts;
10350 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_DRAIN];
10352 /* All other cycles hit here 2-7 */
10353 if ((old_state == BBR_SUB_DRAIN) && bbr->rc_hit_state_1) {
10355 (bbr->rc_use_google == 0) &&
10356 (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {
10357 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
10360 if ((cts - bbr->r_ctl.rc_bbr_state_time) > bbr_get_rtt(bbr, BBR_RTT_PROP))
10361 bbr->r_ctl.rc_exta_time_gd += ((cts - bbr->r_ctl.rc_bbr_state_time) -
10364 bbr->r_ctl.rc_exta_time_gd = 0;
10365 if (bbr->r_ctl.rc_exta_time_gd) {
10366 bbr->r_ctl.rc_level_state_extra = bbr->r_ctl.rc_exta_time_gd;
10368 bbr->r_ctl.rc_level_state_extra /= 7;
10369 if (bbr_rand_ot && bbr->r_ctl.rc_level_state_extra) {
10375 bbr->r_ctl.rc_bbr_state_atflight = max(1, cts);
10376 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[bbr_state_val(bbr)];
10378 if (bbr->rc_use_google) {
10379 bbr->r_ctl.rc_bbr_state_atflight = max(1, cts);
10381 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
10382 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain;
10386 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
10389 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
10390 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {
10393 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
10396 bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff;
10399 (bbr->rc_use_google == 0) &&
10402 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
10403 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
10406 bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered +
10407 ctf_flight_size(bbr->rc_tp,
10408 (bbr->r_ctl.rc_sacked +
10409 bbr->r_ctl.rc_lost_bytes)));
10413 if (bbr->rc_lt_use_bw) {
10414 /* In policed mode we clamp pacing_gain to BBR_UNIT */
10415 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
10418 if (bbr->rc_use_google)
10420 bbr->r_ctl.gain_epoch = cts;
10421 bbr->r_ctl.rc_bbr_state_time = cts;
10422 bbr->r_ctl.substate_pe = bbr->r_ctl.rc_pkt_epoch;
10430 (bbr->r_ctl.rc_flight_at_input <= bbr->r_ctl.rc_target_at_state)) {
10434 if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time)) {
10437 if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_get_rtt(bbr, BBR_RTT_PROP)) {
10450 * we stay in GAIN (gain-to-target).
10454 if (bbr->r_ctl.rc_target_at_state > bbr->r_ctl.rc_flight_at_input) {
10468 if (bbr->rc_use_google) {
10485 if (bbr->r_ctl.rc_bbr_state_atflight == 0) {
10487 flight = ctf_flight_size(bbr->rc_tp,
10488 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
10489 if (bbr_sub_drain_slam_cwnd && bbr->rc_hit_state_1) {
10491 if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state) {
10492 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
10497 bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered + flight);
10500 if (TSTMP_GT(cts, bbr->r_ctl.gain_epoch) &&
10501 (((cts - bbr->r_ctl.gain_epoch) > bbr_get_rtt(bbr, BBR_RTT_PROP)) ||
10502 (flight >= bbr->r_ctl.flightsize_at_drain))) {
10510 bbr->r_ctl.flightsize_at_drain = flight;
10515 bbr->r_ctl.rc_bbr_hptsi_gain *= bbr_drain_drop_mul;
10516 bbr->r_ctl.rc_bbr_hptsi_gain /= bbr_drain_drop_div;
10519 bbr->r_ctl.rc_bbr_hptsi_gain *= 4;
10520 bbr->r_ctl.rc_bbr_hptsi_gain /= 5;
10522 if (bbr->r_ctl.rc_bbr_hptsi_gain <= bbr_drain_floor) {
10524 bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1);
10531 bbr->r_ctl.gain_epoch = cts;
10533 if (flight <= bbr->r_ctl.rc_target_at_state) {
10535 (bbr->rc_use_google == 0) &&
10536 (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {
10537 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
10540 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);
10545 if (bbr->r_ctl.rc_lost > bbr->r_ctl.bbr_lost_at_state) {
10546 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);
10549 if ((ctf_outstanding(bbr->rc_tp) >= bbr->r_ctl.rc_target_at_state) ||
10550 ((ctf_outstanding(bbr->rc_tp) + bbr->rc_tp->t_maxseg - 1) >=
10551 bbr->rc_tp->snd_wnd)) {
10552 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);
10568 if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time))
10570 if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_cur_cycle_time) {
10571 /* Less than a full time-period has passed */
10574 if (bbr->r_ctl.rc_level_state_extra &&
10576 ((cts - bbr->r_ctl.rc_bbr_state_time) <
10577 (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) {
10578 /* Less than a full time-period + extra has passed */
10582 bbr->r_ctl.rc_level_state_extra &&
10584 ((cts - bbr->r_ctl.rc_bbr_state_time) <
10585 (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) {
10586 /* Less than a full time-period + extra has passed */
10597 if (bbr->rc_use_google) {
10601 mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options),
10602 bbr->r_ctl.rc_pace_max_segs);
10618 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&
10619 ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) {
10620 /* Special case using old probe-rtt method */
10621 tar = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
10624 /* Non-probe-rtt case and reduced probe-rtt */
10625 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&
10626 (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT)) {
10628 tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain);
10630 } else if ((bbr_target_is_bbunit) || bbr->rc_use_google) {
10640 * for non-google mode and default (non-configured).
10643 if (bbr->r_ctl.rc_bbr_hptsi_gain < bbr_hptsi_gain[BBR_SUB_DRAIN]) {
10647 tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain);
10653 bbr->r_ctl.rc_target_at_state = tar;
10662 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
10663 bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp,
10664 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
10665 bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.flightsize_at_drain
10666 + bbr->r_ctl.rc_delivered);
10668 if (bbr->rc_use_google || bbr_probertt_sets_rtt)
10669 bbr->rc_prtt_set_ts = 1;
10670 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
10671 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
10672 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
10675 bbr->r_ctl.rc_rtt_shrinks = cts;
10676 bbr->r_ctl.last_in_probertt = cts;
10677 bbr->r_ctl.rc_probertt_srttchktim = cts;
10678 bbr->r_ctl.rc_bbr_state_time = cts;
10679 bbr->rc_bbr_state = BBR_STATE_PROBE_RTT;
10683 bbr->rc_hit_state_1 &&
10684 (bbr->rc_use_google == 0) &&
10686 if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_saved_cwnd)
10687 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
10689 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
10691 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
10692 if ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google){
10693 /* Set to the non-configurable default of 4 (PROBE_RTT_MIN) */
10694 bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
10696 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
10697 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
10698 bbr_log_set_of_state_target(bbr, bbr->rc_tp->snd_cwnd, __LINE__, 6);
10699 bbr->r_ctl.rc_target_at_state = bbr->rc_tp->snd_cwnd;
10706 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val;
10707 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
10710 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
10711 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
10715 if (ctf_flight_size(bbr->rc_tp,
10716 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <=
10717 bbr->r_ctl.rc_target_at_state) {
10719 bbr->r_ctl.rc_bbr_enters_probertt = cts;
10722 bbr->r_ctl.rc_bbr_enters_probertt = 0;
10724 bbr->r_ctl.rc_pe_of_prtt = bbr->r_ctl.rc_pkt_epoch;
10734 * Sanity check on probe-rtt intervals.
10736 * against new-reno flows with huge buffers
10737 * our rtt-prop interval could come to dominate
10742 (bbr->rc_use_google == 0)) {
10746 /* Are we to small and go into probe-rtt to often? */
10752 newval = cur_rttp + (fval - bbr_rtt_probe_limit);
10761 if (cur_rttp > bbr->r_ctl.rc_probertt_int) {
10762 bbr->r_ctl.rc_probertt_int = cur_rttp;
10763 reset_time_small(&bbr->r_ctl.rc_rttprop, newval);
10770 if (bbr->r_ctl.rc_probertt_int > bbr_rtt_probe_limit) {
10776 bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;
10777 reset_time_small(&bbr->r_ctl.rc_rttprop,
10786 if (cur_rttp < bbr->r_ctl.rc_probertt_int) {
10788 bbr->r_ctl.rc_probertt_int = cur_rttp;
10789 reset_time_small(&bbr->r_ctl.rc_rttprop, newval);
10803 /* Exit probe-rtt */
10805 if (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd) {
10806 tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
10810 bbr->rc_hit_state_1 = 0;
10811 bbr->r_ctl.rc_rtt_shrinks = cts;
10812 bbr->r_ctl.last_in_probertt = cts;
10814 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
10815 bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp,
10816 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
10817 bbr->r_ctl.rc_delivered);
10818 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
10821 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
10822 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
10824 if (bbr->rc_filled_pipe) {
10826 bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
10827 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
10828 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain;
10833 bbr->rc_bbr_state = BBR_STATE_STARTUP;
10834 bbr->r_ctl.rc_bbr_state_time = cts;
10838 * the number of pe's we were in probe-rtt
10842 bbr->r_ctl.rc_bbr_last_startup_epoch += (bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_pe_of_prtt);
10843 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
10845 if (bbr->r_ctl.rc_lost &&
10847 (bbr->rc_use_google == 0))
10848 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower;
10850 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;
10851 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;
10855 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
10856 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 0);
10864 if ((bbr->rc_past_init_win == 1) &&
10865 (bbr->rc_in_persist == 0) &&
10866 (bbr_calc_time(cts, bbr->r_ctl.rc_rtt_shrinks) >= bbr->r_ctl.rc_probertt_int)) {
10870 (bbr->rc_in_persist == 0) &&
10871 (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) &&
10872 ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) {
10884 * Need to be on a pkt-epoch to continue.
10889 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *
10890 (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;
10892 bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch;
10893 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
10894 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3);
10895 bbr->r_ctl.rc_bbr_lastbtlbw = btlbw;
10897 if ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS)
10899 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
10900 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8);
10912 if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) &&
10913 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {
10926 if (bbr->rc_use_google)
10929 if ((bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) &&
10932 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower;
10936 * Need to be on a pkt-epoch to continue.
10946 if (bbr->r_ctl.rc_pkt_epoch_rtt &&
10947 bbr->r_ctl.startup_last_srtt &&
10948 (bbr->r_ctl.rc_pkt_epoch_rtt > bbr->r_ctl.startup_last_srtt)) {
10949 delta = bbr->r_ctl.rc_pkt_epoch_rtt - bbr->r_ctl.startup_last_srtt;
10950 rtt_gain = (delta * 100) / bbr->r_ctl.startup_last_srtt;
10953 if ((bbr->r_ctl.startup_last_srtt == 0) ||
10954 (bbr->r_ctl.rc_pkt_epoch_rtt < bbr->r_ctl.startup_last_srtt))
10956 bbr->r_ctl.startup_last_srtt = bbr->r_ctl.rc_pkt_epoch_rtt;
10958 if ((bbr->r_ctl.rc_lost == 0) &&
10965 if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch)
10966 bbr->r_ctl.rc_bbr_last_startup_epoch++;
10968 delta, bbr->r_ctl.startup_last_srtt, 10);
10972 if ((bbr->r_ctl.r_measurement_count == bbr->r_ctl.last_startup_measure) &&
10973 (bbr->r_ctl.rc_lost_at_startup == bbr->r_ctl.rc_lost) &&
10974 (!IN_RECOVERY(bbr->rc_tp->t_flags))) {
10979 * the number of non-gain we have already accumulated.
10981 if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch)
10982 bbr->r_ctl.rc_bbr_last_startup_epoch++;
10983 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
10984 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 9);
10988 if (bbr->r_ctl.rc_lost_at_startup > bbr->r_ctl.rc_lost)
10989 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
10990 bbr->r_ctl.last_startup_measure = bbr->r_ctl.r_measurement_count;
10992 if (bbr->r_ctl.rc_bbr_hptsi_gain == bbr_startup_lower)
10993 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *
10994 (uint64_t)bbr_low_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;
10996 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *
10997 (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;
10999 if (btlbw > bbr->r_ctl.rc_bbr_lastbtlbw)
11000 bbr->r_ctl.rc_bbr_lastbtlbw = btlbw;
11002 bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch;
11004 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
11005 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11006 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3);
11008 if ((bbr->rc_loss_exit &&
11009 (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) &&
11010 (bbr->r_ctl.rc_pkt_epoch_loss_rate > bbr_startup_loss_thresh)) &&
11011 ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS)) {
11019 if ((ctf_flight_size(bbr->rc_tp,
11020 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
11021 (2 * max(bbr->r_ctl.rc_pace_max_segs, bbr->rc_tp->t_maxseg))) <= bbr->rc_tp->snd_wnd) {
11023 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11024 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 4);
11027 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
11028 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11029 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 5);
11032 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
11033 if (((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS) ||
11039 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11040 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8);
11051 if ((bbr->rc_bbr_state != BBR_STATE_STARTUP) &&
11052 (bbr->rc_bbr_state != BBR_STATE_DRAIN) &&
11053 (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) &&
11054 (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) &&
11055 (bbr->rc_bbr_state != BBR_STATE_PROBE_BW)) {
11057 panic("Unknown BBR state %d?\n", bbr->rc_bbr_state);
11060 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
11065 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11066 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 6);
11067 bbr->rc_filled_pipe = 1;
11068 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
11069 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
11070 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
11071 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
11074 if (bbr->rc_no_pacing)
11075 bbr->rc_no_pacing = 0;
11076 bbr->r_ctl.rc_bbr_state_time = cts;
11077 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_drain_pg;
11078 bbr->rc_bbr_state = BBR_STATE_DRAIN;
11080 if ((bbr->rc_use_google == 0) &&
11082 /* Here we don't have to worry about probe-rtt */
11083 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
11084 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
11087 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain;
11089 if (ctf_flight_size(bbr->rc_tp,
11090 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <=
11091 bbr->r_ctl.rc_target_at_state) {
11096 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
11098 bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
11102 } else if (bbr->rc_bbr_state == BBR_STATE_IDLE_EXIT) {
11106 tp = bbr->rc_tp;
11108 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11109 if (inflight >= bbr->r_ctl.rc_target_at_state) {
11111 bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
11112 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
11113 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
11119 bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff;
11120 bbr->rc_bbr_substate = BBR_SUB_LEVEL6;
11123 } else if (bbr->rc_bbr_state == BBR_STATE_DRAIN) {
11124 /* Has in-flight reached the bdp (or less)? */
11128 tp = bbr->rc_tp;
11130 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11131 if ((bbr->rc_use_google == 0) &&
11133 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
11135 * Here we don't have to worry about probe-rtt
11136 * re-slam it, but keep it slammed down.
11138 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
11141 if (inflight <= bbr->r_ctl.rc_target_at_state) {
11143 bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
11144 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
11145 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
11148 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
11149 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
11151 if ((bbr->rc_use_google == 0) &&
11153 (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {
11155 tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
11158 /* Setup probe-rtt has being done now RRS-HERE */
11159 bbr->r_ctl.rc_rtt_shrinks = cts;
11160 bbr->r_ctl.last_in_probertt = cts;
11162 /* Randomly pick a sub-state */
11163 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
11167 } else if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) {
11170 flight = ctf_flight_size(bbr->rc_tp,
11171 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11172 bbr->r_ctl.r_app_limited_until = (flight + bbr->r_ctl.rc_delivered);
11173 if (((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google) &&
11174 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
11178 bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
11181 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
11182 /* Re-slam it */
11183 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
11186 if (bbr->r_ctl.rc_bbr_enters_probertt == 0) {
11188 if (flight <= bbr->r_ctl.rc_target_at_state) {
11190 bbr->r_ctl.rc_bbr_enters_probertt = cts;
11192 if (bbr->r_ctl.rc_bbr_enters_probertt == 0)
11193 bbr->r_ctl.rc_bbr_enters_probertt = 1;
11194 if (bbr->rc_use_google == 0) {
11199 if (bbr->r_ctl.bbr_rttprobe_gain_val)
11200 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val;
11202 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
11205 if ((bbr->r_ctl.rc_bbr_enters_probertt == 0) &&
11206 (bbr->rc_use_google == 0) &&
11207 bbr->r_ctl.bbr_rttprobe_gain_val &&
11208 (((cts - bbr->r_ctl.rc_probertt_srttchktim) > bbr_get_rtt(bbr, bbr_drain_rtt)) ||
11209 (flight >= bbr->r_ctl.flightsize_at_drain))) {
11220 bbr->r_ctl.flightsize_at_drain = flight;
11221 bbr->r_ctl.rc_probertt_srttchktim = cts;
11222 red = max((bbr->r_ctl.bbr_rttprobe_gain_val / 10), 1);
11223 if ((bbr->r_ctl.rc_bbr_hptsi_gain - red) > max(bbr_drain_floor, 1)) {
11225 bbr->r_ctl.rc_bbr_hptsi_gain -= red;
11227 } else if (bbr->r_ctl.rc_bbr_hptsi_gain > max(bbr_drain_floor, 1)) {
11229 bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1);
11233 bbr->r_ctl.rc_bbr_hptsi_gain = max((bbr_drain_floor-1), 1);
11237 if (bbr->r_ctl.rc_bbr_enters_probertt &&
11238 (TSTMP_GT(cts, bbr->r_ctl.rc_bbr_enters_probertt)) &&
11239 ((cts - bbr->r_ctl.rc_bbr_enters_probertt) >= bbr_rtt_probe_time)) {
11241 bbr_exit_probe_rtt(bbr->rc_tp, bbr, cts);
11243 } else if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {
11244 if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) &&
11245 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {
11267 if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP)) {
11272 bbr_state_change(bbr, cts, epoch, bbr->rc_is_pkt_epoch_now, losses);
11294 nsegs = max(1, m->m_pkthdr.lro_nsegs);
11295 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
11301 * If this is either a state-changing packet or current state isn't
11308 KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN",
11310 KASSERT(tp->t_state != TCPS_TIME_WAIT, ("%s: TCPS_TIME_WAIT",
11313 tp->t_rcvtime = ticks;
11315 * Unscale the window into a 32-bit value. For the SYN_SENT state
11318 tiwin = th->th_win << tp->snd_scale;
11320 stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_FRWIN, tiwin);
11323 if (m->m_flags & M_TSTMP) {
11328 bbr->rc_tv.tv_sec = ts.tv_sec;
11329 bbr->rc_tv.tv_usec = ts.tv_nsec / 1000;
11330 bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usectick(&bbr->rc_tv);
11331 } else if (m->m_flags & M_TSTMP_LRO) {
11336 bbr->rc_tv.tv_sec = ts.tv_sec;
11337 bbr->rc_tv.tv_usec = ts.tv_nsec / 1000;
11338 bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usectick(&bbr->rc_tv);
11343 bbr->r_ctl.rc_rcvtime = lcts = cts = tcp_get_usecs(&bbr->rc_tv);
11349 (th->th_off << 2) - sizeof(struct tcphdr),
11351 if (tp->t_flags2 & TF2_PROC_SACK_PROHIBIT) {
11366 if ((tp->t_flags & TF_RCVD_TSTMP) && !(to.to_flags & TOF_TS) &&
11378 to.to_tsecr -= tp->ts_offset;
11379 if (TSTMP_GT(to.to_tsecr, tcp_tv_to_mssectick(&bbr->rc_tv)))
11386 if (bbr->r_state == 0) {
11394 if (bbr->rc_inp == NULL) {
11395 bbr->rc_inp = inp;
11401 if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) {
11403 (tp->t_flags & TF_REQ_SCALE)) {
11404 tp->t_flags |= TF_RCVD_SCALE;
11405 tp->snd_scale = to.to_wscale;
11407 tp->t_flags &= ~TF_REQ_SCALE;
11412 tp->snd_wnd = th->th_win;
11414 (tp->t_flags & TF_REQ_TSTMP)) {
11415 tp->t_flags |= TF_RCVD_TSTMP;
11416 tp->ts_recent = to.to_tsval;
11417 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
11419 tp->t_flags &= ~TF_REQ_TSTMP;
11422 if ((tp->t_flags & TF_SACK_PERMIT) &&
11424 tp->t_flags &= ~TF_SACK_PERMIT;
11425 if (tp->t_flags & TF_FASTOPEN) {
11432 if ((inp->inp_vflag & INP_IPV6) != 0)
11448 if ((tp->t_flags & TF_SACK_PERMIT) == 0) {
11451 (*tp->t_fb->tfb_tcp_do_segment)(tp, m, th, drop_hdrlen,
11456 bbr->r_is_v6 = (inp->inp_vflag & INP_IPV6) != 0;
11458 sack_filter_clear(&bbr->r_ctl.bbr_sf, th->th_ack);
11469 * always. All other times (timers etc) we must have a rack-state
11474 if (bbr->r_state != tp->t_state)
11477 if (SEQ_GT(th->th_ack, tp->snd_una) && (rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map)) != NULL)
11479 prev_state = bbr->r_state;
11480 bbr->rc_ack_was_delayed = 0;
11481 lost = bbr->r_ctl.rc_lost;
11482 bbr->rc_is_pkt_epoch_now = 0;
11483 if (m->m_flags & (M_TSTMP|M_TSTMP_LRO)) {
11487 bbr->r_ctl.rc_ack_hdwr_delay = lcts - cts;
11488 bbr->rc_ack_was_delayed = 1;
11489 if (TSTMP_GT(bbr->r_ctl.rc_ack_hdwr_delay,
11490 bbr->r_ctl.highest_hdwr_delay))
11491 bbr->r_ctl.highest_hdwr_delay = bbr->r_ctl.rc_ack_hdwr_delay;
11493 bbr->r_ctl.rc_ack_hdwr_delay = 0;
11494 bbr->rc_ack_was_delayed = 0;
11497 bbr->r_ctl.rc_ack_hdwr_delay = 0;
11498 bbr->rc_ack_was_delayed = 0;
11507 * If a segment with the ACK-bit set arrives in the SYN-SENT state
11510 if ((tp->t_state == TCPS_SYN_SENT) && (thflags & TH_ACK) &&
11511 (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) {
11516 if (tiwin > bbr->r_ctl.rc_high_rwnd)
11517 bbr->r_ctl.rc_high_rwnd = tiwin;
11518 bbr->r_ctl.rc_flight_at_input = ctf_flight_size(tp,
11519 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11520 bbr->rtt_valid = 0;
11522 bbr->rc_ts_valid = 1;
11523 bbr->r_ctl.last_inbound_ts = to.to_tsval;
11525 bbr->rc_ts_valid = 0;
11526 bbr->r_ctl.last_inbound_ts = 0;
11528 retval = (*bbr->r_substate) (m, th, so,
11542 if (bbr->rc_is_pkt_epoch_now)
11544 bbr_check_bbr_for_state(bbr, cts, __LINE__, (bbr->r_ctl.rc_lost - lost));
11546 if ((bbr->r_wanted_output != 0) ||
11547 (tp->t_flags & TF_ACKNOW)) {
11549 bbr->rc_output_starts_timer = 0;
11557 ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) == 0) &&
11558 (SEQ_GT(tp->snd_max, tp->snd_una) ||
11559 (tp->t_flags & TF_DELACK) ||
11560 ((V_tcp_always_keepalive || bbr->rc_inp->inp_socket->so_options & SO_KEEPALIVE) &&
11561 (tp->t_state <= TCPS_CLOSING)))) {
11566 if ((tp->snd_max == tp->snd_una) &&
11567 ((tp->t_flags & TF_DELACK) == 0) &&
11569 (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
11578 if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
11579 (TSTMP_GT(lcts, bbr->rc_pacer_started))) {
11582 del = lcts - bbr->rc_pacer_started;
11583 if (bbr->r_ctl.rc_last_delay_val > del) {
11585 bbr->r_ctl.rc_last_delay_val -= del;
11586 bbr->rc_pacer_started = lcts;
11589 bbr->r_ctl.rc_last_delay_val = 0;
11596 bbr_start_hpts_timer(bbr, tp, cts, 8, bbr->r_ctl.rc_last_delay_val,
11599 } else if ((bbr->rc_output_starts_timer == 0) && (nxt_pkt == 0)) {
11601 bbr_timer_audit(tp, bbr, lcts, &so->so_snd);
11604 if ((nxt_pkt == 0) && (tp->t_flags2 & TF2_HPTS_CALLS))
11605 tp->t_flags2 &= ~TF2_HPTS_CALLS;
11607 if (bbr->r_state != tp->t_state)
11612 bbr->r_wanted_output = 0;
11625 if (!STAILQ_EMPTY(&tp->t_inqueue)) {
11631 if (m->m_flags & M_TSTMP_LRO) {
11655 if (ctf_outstanding(tp) >= tp->snd_wnd) {
11656 /* We never want to go over our peers rcv-window */
11661 flight = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11666 * >= tp->snd_wnd).
11670 len = sendwin - flight;
11671 if ((len + ctf_outstanding(tp)) > tp->snd_wnd) {
11673 len = tp->snd_wnd - ctf_outstanding(tp);
11680 len = avail - sb_offset;
11693 if (rsm->r_flags & BBR_TLP) {
11702 tp->t_sndrexmitpack++;
11706 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB,
11711 * Logs in 0 - 8, 8 is all non probe_bw states 0-7 is
11712 * sub-state
11714 counter_u64_add(bbr_state_lost[rsm->r_bbr_state], len);
11715 if (bbr->rc_bbr_state != BBR_STATE_PROBE_BW) {
11717 counter_u64_add(bbr_state_resend[bbr->rc_bbr_state], len);
11720 * Log our probe state 3, and log also 5-13 to show
11721 * us the recovery sub-state for the send. This
11742 stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB,
11751 if (bbr->rc_filled_pipe && bbr_target_cwnd_mult_limit && (bbr->rc_use_google == 0)) {
11762 if (tp->snd_cwnd > target)
11763 tp->snd_cwnd = target;
11773 * account that we are limited by TCP_MAXWIN << tp->rcv_scale.
11779 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) {
11780 oldwin = (tp->rcv_adv - tp->rcv_nxt);
11782 adv -= oldwin;
11795 if (oldwin >> tp->rcv_scale >= (adv + oldwin) >> tp->rcv_scale)
11799 (adv >= (so->so_rcv.sb_hiwat / 4) ||
11800 recwin <= (so->so_rcv.sb_hiwat / 8) ||
11801 so->so_rcv.sb_hiwat <= 8 * maxseg)) {
11804 if (2 * adv >= (int32_t) so->so_rcv.sb_hiwat)
11812 * if the TCB was on the hpts. A non-zero return
11870 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
11872 memcpy(&bbr->rc_tv, tv, sizeof(struct timeval));
11873 cts = tcp_tv_to_usectick(&bbr->rc_tv);
11874 inp = bbr->rc_inp;
11875 hpts_calling = !!(tp->t_flags2 & TF2_HPTS_CALLS);
11876 tp->t_flags2 &= ~TF2_HPTS_CALLS;
11877 so = inp->inp_socket;
11878 sb = &so->so_snd;
11879 if (tp->t_nic_ktls_xmit)
11884 maxseg = tp->t_maxseg - bbr->rc_last_options;
11889 pace_max_segs = bbr->r_ctl.rc_pace_max_segs;
11893 if (tp->t_flags & TF_TOE)
11898 if (bbr->r_state) {
11900 isipv6 = bbr->r_is_v6;
11902 isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
11905 if (((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) &&
11911 if ((tp->t_flags & TF_ACKNOW) == 0) {
11916 * whatever timer is running (KEEP/DEL-ACK?) and
11921 recwin = lmin(lmax(sbspace(&so->so_rcv), 0),
11922 (long)TCP_MAXWIN << tp->rcv_scale);
11924 ((tcp_outflags[tp->t_state] & TH_RST) == 0) &&
11925 ((sbavail(sb) + ((tcp_outflags[tp->t_state] & TH_FIN) ? 1 : 0)) <=
11926 (tp->snd_max - tp->snd_una))) {
11930 * let the timer-run off.
11938 if (bbr->r_ctl.rc_last_delay_val) {
11940 if (SEQ_GT(cts, bbr->rc_pacer_started))
11941 delay_calc = cts - bbr->rc_pacer_started;
11942 if (delay_calc >= bbr->r_ctl.rc_last_delay_val)
11943 delay_calc -= bbr->r_ctl.rc_last_delay_val;
11948 if ((bbr->r_timer_override) ||
11949 (tp->t_state < TCPS_ESTABLISHED)) {
11954 if ((bbr->r_ctl.rc_last_delay_val) &&
11955 (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
11964 bbr->r_ctl.rc_last_delay_val = 0;
11966 } else if (tp->t_state == TCPS_CLOSED) {
11967 bbr->r_ctl.rc_last_delay_val = 0;
11979 bbr->rc_cwnd_limited = 0;
11980 if (bbr->r_ctl.rc_last_delay_val) {
11982 if (SEQ_GT(cts, bbr->rc_pacer_started))
11983 delay_calc = cts - bbr->rc_pacer_started;
11986 if (delay_calc >= bbr->r_ctl.rc_last_delay_val)
11988 delay_calc -= bbr->r_ctl.rc_last_delay_val;
11996 bbr->r_ctl.rc_agg_early += (bbr->r_ctl.rc_last_delay_val - delay_calc);
11997 bbr->r_agg_early_set = 1;
11998 if (bbr->r_ctl.rc_hptsi_agg_delay) {
11999 if (bbr->r_ctl.rc_hptsi_agg_delay >= bbr->r_ctl.rc_agg_early) {
12001 bbr->r_ctl.rc_hptsi_agg_delay -= bbr->r_ctl.rc_agg_early;
12002 bbr->r_agg_early_set = 0;
12003 bbr->r_ctl.rc_agg_early = 0;
12005 bbr->r_ctl.rc_agg_early -= bbr->r_ctl.rc_hptsi_agg_delay;
12006 bbr->r_ctl.rc_hptsi_agg_delay = 0;
12009 merged_val = bbr->rc_pacer_started;
12011 merged_val |= bbr->r_ctl.rc_last_delay_val;
12013 bbr->r_ctl.rc_agg_early, cts, delay_calc, merged_val,
12014 bbr->r_agg_early_set, 3);
12015 bbr->r_ctl.rc_last_delay_val = 0;
12021 if (bbr->r_agg_early_set)
12022 bbr->r_ctl.rc_agg_early = 0;
12023 bbr->r_agg_early_set = 0;
12032 if ((bbr->r_ctl.rc_hptsi_agg_delay + delay_calc) < bbr->r_ctl.rc_hptsi_agg_delay)
12033 bbr->r_ctl.rc_hptsi_agg_delay = 0xffffffff;
12035 bbr->r_ctl.rc_hptsi_agg_delay += delay_calc;
12037 sendwin = min(tp->snd_wnd, tp->snd_cwnd);
12038 if ((tp->snd_una == tp->snd_max) &&
12039 (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) &&
12044 * suite of states or a fast-ramp up.
12047 cts, bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time));
12055 if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
12068 bbr->rc_tp->t_flags2 &= ~TF2_MBUF_QUEUE_READY;
12070 (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
12071 bbr->r_ctl.rc_last_delay_val = 0;
12073 bbr->r_timer_override = 0;
12074 bbr->r_wanted_output = 0;
12079 if ((tp->t_flags & TF_FASTOPEN) &&
12080 ((tp->t_state == TCPS_SYN_RECEIVED) ||
12081 (tp->t_state == TCPS_SYN_SENT)) &&
12082 SEQ_GT(tp->snd_max, tp->snd_una) && /* initial SYN or SYN|ACK sent */
12083 (tp->t_rxtshift == 0)) { /* not a retransmit */
12092 if (bbr->rc_use_google == 0)
12111 sendwin = min(tp->snd_wnd, tp->snd_cwnd);
12112 sb_offset = tp->snd_max - tp->snd_una;
12113 flags = tcp_outflags[tp->t_state];
12122 while (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) {
12128 tot_len = tp->t_maxseg;
12134 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next);
12135 bbr->r_ctl.rc_free_cnt++;
12139 if (bbr->r_ctl.rc_resend == NULL) {
12141 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);
12142 if (bbr->r_ctl.rc_resend) {
12146 bbr_cong_signal(tp, NULL, CC_NDUPACK, bbr->r_ctl.rc_resend);
12149 if (bbr->r_ctl.rc_resend) {
12150 rsm = bbr->r_ctl.rc_resend;
12154 /* Remove any TLP flags its a RACK or T-O */
12155 rsm->r_flags &= ~BBR_TLP;
12156 bbr->r_ctl.rc_resend = NULL;
12157 if (SEQ_LT(rsm->r_start, tp->snd_una)) {
12160 tp, bbr, rsm, rsm->r_start, tp->snd_una);
12168 if (rsm->r_flags & BBR_HAS_SYN) {
12173 rsm->r_start++;
12174 if (rsm->r_start == rsm->r_end) {
12179 rsm->r_flags &= ~BBR_HAS_SYN;
12180 len = rsm->r_end - rsm->r_start;
12189 len = rsm->r_end - rsm->r_start;
12190 if ((bbr->rc_resends_use_tso == 0) &&
12195 sb_offset = rsm->r_start - tp->snd_una;
12207 } else if (bbr->r_ctl.rc_tlp_send) {
12212 rsm = bbr->r_ctl.rc_tlp_send;
12213 bbr->r_ctl.rc_tlp_send = NULL;
12215 len = rsm->r_end - rsm->r_start;
12216 if ((bbr->rc_resends_use_tso == 0) && (len > maxseg))
12219 if (SEQ_GT(tp->snd_una, rsm->r_start)) {
12222 tp, bbr, tp->snd_una, rsm, rsm->r_start);
12229 sb_offset = rsm->r_start - tp->snd_una;
12238 (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {
12240 if (!bbr->alloc_limit_reported) {
12241 bbr->alloc_limit_reported = 1;
12247 if (rsm && SEQ_LT(rsm->r_start, tp->snd_una)) {
12256 if (tp->t_flags & TF_NEEDFIN && (rsm == NULL))
12258 if (tp->t_flags & TF_NEEDSYN)
12261 if (rsm && (rsm->r_flags & BBR_HAS_FIN)) {
12263 len--;
12279 end_rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext);
12290 * fast-retransmit because TCP will reset snd_nxt to snd_max after
12291 * the fast-retransmit.
12293 * In the normal retransmit-FIN-only case, however, snd_nxt will be
12304 if (SEQ_GT(tp->snd_max, tp->snd_una))
12305 sb_offset = tp->snd_max - tp->snd_una;
12308 if (bbr->rc_tlp_new_data) {
12315 if (tlplen > (uint32_t)(avail - sb_offset)) {
12316 tlplen = (uint32_t)(avail - sb_offset);
12318 if (tlplen > tp->snd_wnd) {
12319 len = tp->snd_wnd;
12323 bbr->rc_tlp_new_data = 0;
12327 (bbr->rc_in_persist == 0) &&
12329 ((avail - sb_offset) >= p_maxseg)) {
12341 if (bbr->rc_in_persist) {
12348 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
12350 len = rsm->r_end - rsm->r_start;
12351 if (rsm->r_flags & BBR_HAS_FIN)
12352 len--;
12353 if ((bbr->rc_resends_use_tso == 0) && (len > maxseg))
12366 sb_offset = rsm->r_start - tp->snd_una;
12387 * SYN-SENT state and if segment contains data and if we don't know
12391 SEQ_GT(tp->snd_max, tp->snd_una)) {
12392 if (tp->t_state != TCPS_SYN_RECEIVED)
12398 if ((tp->t_flags & TF_FASTOPEN) &&
12399 (tp->t_state == TCPS_SYN_RECEIVED))
12401 sb_offset--, len++;
12409 len--;
12416 if ((flags & TH_SYN) && (tp->t_flags & TF_NOOPT)) {
12423 * - When retransmitting SYN|ACK on a passively-created socket
12424 * - When retransmitting SYN on an actively created socket
12425 * - When sending a zero-length cookie (cookie request) on an
12427 * - When the socket is in the CLOSED state (RST is being sent)
12429 if ((tp->t_flags & TF_FASTOPEN) &&
12430 (((flags & TH_SYN) && (tp->t_rxtshift > 0)) ||
12431 ((tp->t_state == TCPS_SYN_SENT) &&
12432 (tp->t_tfo_client_cookie_len == 0)) ||
12438 /* Without fast-open there should never be data sent on a SYN */
12439 if ((flags & TH_SYN) && !(tp->t_flags & TF_FASTOPEN))
12453 * 0-byte window. This makes sure the persist timer is set
12458 if ((tp->snd_wnd == 0) &&
12459 (TCPS_HAVEESTABLISHED(tp->t_state)) &&
12460 (tp->snd_una == tp->snd_max) &&
12470 (len < bbr->r_ctl.rc_pace_max_segs)) {
12476 if ((tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
12477 (TCPS_HAVEESTABLISHED(tp->t_state)) &&
12478 (len < (int)(sbavail(sb) - sb_offset))) {
12487 if (tp->snd_max == tp->snd_una)
12489 } else if ((tp->snd_cwnd >= bbr->r_ctl.rc_pace_max_segs) &&
12490 (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12491 bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) &&
12492 (len < (int)(sbavail(sb) - sb_offset)) &&
12504 bbr->rc_cwnd_limited = 1;
12506 } else if (((tp->snd_wnd - ctf_outstanding(tp)) <
12507 min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
12508 (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12509 bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) &&
12510 (len < (int)(sbavail(sb) - sb_offset)) &&
12511 (TCPS_HAVEESTABLISHED(tp->t_state))) {
12529 if (bbr->rc_in_persist &&
12532 (len < min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs))) {
12540 sbleft = sbavail(sb) - sb_offset;
12543 if (sbleft >= min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs)) {
12553 * presence of TCP-MD5, SACK retransmits, SACK advertizements and IP
12570 if (inp->inp_options)
12571 ipoptlen = inp->inp_options->m_len -
12577 * Pre-calculate here as we save another lookup into the darknesses
12595 if ((tp->t_flags & TF_TSO) && V_tcp_do_tso &&
12597 (tp->t_port == 0) &&
12598 ((tp->t_flags & TF_SIGNATURE) == 0) &&
12602 recwin = lmin(lmax(sbspace(&so->so_rcv), 0),
12603 (long)TCP_MAXWIN << tp->rcv_scale);
12606 * conditions when len is non-zero:
12608 * - We have a full segment (or more with TSO) - This is the last
12610 * NODELAY - we've timed out (e.g. persist timer) - we have more
12612 * limited the window size) - we need to retransmit
12627 if (((tp->t_flags & TF_MORETOCOME) == 0) && /* normal case */
12628 ((tp->t_flags & TF_NODELAY) ||
12629 ((uint32_t)len + (uint32_t)sb_offset) >= sbavail(&so->so_snd)) &&
12630 (tp->t_flags & TF_NOPUSH) == 0) {
12633 if ((tp->snd_una == tp->snd_max) && len) { /* Nothing outstanding */
12636 if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) {
12664 * pending (it will get piggy-backed on it) or the remote side
12665 * already has done a half-close and won't send more data. Skip
12666 * this if the connection is in T/TCP half-open state.
12668 if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN) &&
12669 !(tp->t_flags & TF_DELACK) &&
12670 !TCPS_HAVERCVDFIN(tp->t_state)) {
12677 * is also a catch-all for the retransmit timer timeout case.
12679 if (tp->t_flags & TF_ACKNOW) {
12686 if ((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0) {
12694 ((tp->t_flags & TF_SENTFIN) == 0)) {
12704 slot = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0);
12705 if (bbr->rc_no_pacing)
12708 if ((ctf_outstanding(tp) + min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) >=
12709 tp->snd_wnd) {
12713 if ((bbr->rc_in_persist == 0) &&
12714 TCPS_HAVEESTABLISHED(tp->t_state) &&
12715 (tp->snd_max == tp->snd_una) &&
12716 sbavail(&so->so_snd)) {
12718 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
12724 } else if ((ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12725 bbr->r_ctl.rc_lost_bytes)) + p_maxseg) >= tp->snd_cwnd) {
12728 bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12729 bbr->r_ctl.rc_lost_bytes)));
12730 bbr->rc_cwnd_limited = 1;
12736 bbr->r_ctl.rc_hptsi_agg_delay = 0;
12737 bbr->r_agg_early_set = 0;
12738 bbr->r_ctl.rc_agg_early = 0;
12739 bbr->r_ctl.rc_last_delay_val = 0;
12740 } else if (bbr->rc_use_google == 0)
12748 bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12749 bbr->r_ctl.rc_lost_bytes)) + bbr->r_ctl.rc_delivered);
12754 bbr->r_ctl.rc_last_delay_val = 0;
12755 bbr->rc_output_starts_timer = 1;
12758 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
12760 tp->snd_nxt = tp->snd_max;
12773 bbr->rc_tlp_in_progress = 0;
12774 bbr->rc_tlp_rtx_out = 0;
12779 bbr->rc_tlp_in_progress = 1;
12785 * This is sub-optimal. We only send a stand alone
12790 if ((len == 0) && ((tp->t_flags & TF_ACKNOW) == 0)) {
12803 (((rsm->r_flags & BBR_HAS_FIN) == 0) &&
12806 len--;
12811 if ((tp->snd_una == tp->snd_max) &&
12812 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {
12822 tp->t_flags2 |= TF2_PLPMTU_MAXSEGSNT;
12824 tp->t_flags2 &= ~TF2_PLPMTU_MAXSEGSNT;
12843 * established connection segments. Options for SYN-ACK segments
12848 if ((tp->t_flags & TF_NOOPT) == 0) {
12851 to.to_mss = tcp_mssopt(&inp->inp_inc);
12852 if (tp->t_port)
12853 to.to_mss -= V_tcp_udp_tunneling_overhead;
12862 if ((tp->t_flags & TF_FASTOPEN) &&
12863 (tp->t_rxtshift == 0)) {
12864 if (tp->t_state == TCPS_SYN_RECEIVED) {
12867 (u_int8_t *)&tp->t_tfo_cookie.server;
12870 } else if (tp->t_state == TCPS_SYN_SENT) {
12872 tp->t_tfo_client_cookie_len;
12874 tp->t_tfo_cookie.client;
12881 if ((flags & TH_SYN) && (tp->t_flags & TF_REQ_SCALE)) {
12882 to.to_wscale = tp->request_r_scale;
12886 if ((tp->t_flags & TF_RCVD_TSTMP) ||
12887 ((flags & TH_SYN) && (tp->t_flags & TF_REQ_TSTMP))) {
12888 to.to_tsval = tcp_tv_to_mssectick(&bbr->rc_tv) + tp->ts_offset;
12889 to.to_tsecr = tp->ts_recent;
12894 if (tp->rfbuf_ts == 0 &&
12895 (so->so_rcv.sb_flags & SB_AUTOSIZE))
12896 tp->rfbuf_ts = tcp_tv_to_mssectick(&bbr->rc_tv);
12900 else if (TCPS_HAVEESTABLISHED(tp->t_state) &&
12901 tp->rcv_numsacks > 0) {
12903 to.to_nsacks = tp->rcv_numsacks;
12904 to.to_sacks = (u_char *)tp->sackblks;
12907 /* TCP-MD5 (RFC2385). */
12908 if (tp->t_flags & TF_SIGNATURE)
12918 if ((tp->t_flags & TF_FASTOPEN) && wanted_cookie &&
12922 if (tp->t_port) {
12935 if (inp->inp_options)
12936 ipoptlen = inp->inp_options->m_len -
12944 if (bbr->rc_last_options != local_options) {
12949 bbr->rc_last_options = local_options;
12951 maxseg = tp->t_maxseg - (ipoptlen + optlen);
12967 if_hw_tsomax = tp->t_tsomax;
12968 if_hw_tsomaxsegcount = tp->t_tsomaxsegcount;
12969 if_hw_tsomaxsegsize = tp->t_tsomaxsegsize;
12979 max_len = (if_hw_tsomax - hdrlen -
12994 len -= moff;
13007 if (optlen + ipoptlen >= tp->t_maxseg) {
13045 if (SEQ_LT(rsm->r_start, tp->snd_una)) {
13047 rsm, tp, bbr, rsm->r_start, tp->snd_una);
13091 m->m_data += max_linkhdr;
13092 m->m_len = hdrlen;
13117 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
13124 if (rsm->r_start != tp->snd_una) {
13132 return (-EFAULT); /* tcp_drop() */
13134 len = rsm->r_end - rsm->r_start;
13142 if (len <= MHLEN - hdrlen - max_linkhdr && !hw_tls) {
13147 m->m_len += len;
13161 tp->snd_una, rsm->r_flags, rsm->r_start,
13167 tp, bbr, len, moff, sbavail(sb), sb_offset, tp->snd_una);
13171 m->m_next = tcp_m_copym(
13189 if (m->m_next == NULL) {
13209 foo = foo->m_next;
13229 if (tp->t_flags & TF_ACKNOW)
13251 m->m_data += max_linkhdr;
13252 m->m_len = hdrlen;
13255 m->m_pkthdr.rcvif = (struct ifnet *)0;
13262 if (tp->t_port) {
13264 udp->uh_sport = htons(V_tcp_udp_tunneling_port);
13265 udp->uh_dport = tp->t_port;
13266 ulen = hdrlen + len - sizeof(struct ip6_hdr);
13267 udp->uh_ulen = htons(ulen);
13272 tcpip_fillheaders(inp, tp->t_port, ip6, th);
13277 if (tp->t_port) {
13279 udp->uh_sport = htons(V_tcp_udp_tunneling_port);
13280 udp->uh_dport = tp->t_port;
13281 ulen = hdrlen + len - sizeof(struct ip);
13282 udp->uh_ulen = htons(ulen);
13287 tcpip_fillheaders(inp, tp->t_port, ip, th);
13303 th->th_seq = htonl(tp->snd_max);
13304 bbr_seq = tp->snd_max;
13307 th->th_seq = htonl(tp->iss);
13308 bbr_seq = tp->iss;
13310 if (flags & TH_FIN && tp->t_flags & TF_SENTFIN) {
13315 th->th_seq = (htonl(tp->snd_max - 1));
13316 bbr_seq = (tp->snd_max - 1);
13319 th->th_seq = htonl(tp->snd_max);
13320 bbr_seq = tp->snd_max;
13332 * might be better to send (tp->snd_una - 1) which
13335 if (tp->t_flags & TF_SENTFIN) {
13336 th->th_seq = htonl(tp->snd_max - 1);
13337 bbr_seq = (tp->snd_max - 1);
13339 th->th_seq = htonl(tp->snd_max);
13340 bbr_seq = tp->snd_max;
13345 th->th_seq = htonl(rsm->r_start);
13346 bbr_seq = rsm->r_start;
13348 th->th_ack = htonl(tp->rcv_nxt);
13351 th->th_off = (sizeof(struct tcphdr) + optlen) >> 2;
13358 if ((flags & TH_RST) || ((recwin < (so->so_rcv.sb_hiwat / 4) &&
13361 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt) &&
13362 recwin < (tp->rcv_adv - tp->rcv_nxt))
13363 recwin = (tp->rcv_adv - tp->rcv_nxt);
13364 if (recwin > TCP_MAXWIN << tp->rcv_scale)
13365 recwin = TCP_MAXWIN << tp->rcv_scale;
13373 th->th_win = htons((u_short)
13374 (min(sbspace(&so->so_rcv), TCP_MAXWIN)));
13377 recwin = roundup2(recwin, 1 << tp->rcv_scale);
13378 th->th_win = htons((u_short)(recwin >> tp->rcv_scale));
13381 * Adjust the RXWIN0SENT flag - indicate that we have advertised a 0
13388 if (th->th_win == 0) {
13389 tp->t_sndzerowin++;
13390 tp->t_flags |= TF_RXWIN0SENT;
13392 tp->t_flags &= ~TF_RXWIN0SENT;
13397 tp->snd_up = tp->snd_una;
13402 m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */
13412 (u_char *)(th + 1) + (to.to_signature - opt)) != 0) {
13428 if (tp->t_port) {
13429 m->m_pkthdr.csum_flags = CSUM_UDP_IPV6;
13430 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
13431 udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0);
13432 th->th_sum = htons(0);
13435 csum_flags = m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;
13436 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
13437 th->th_sum = in6_cksum_pseudo(ip6, sizeof(struct tcphdr) +
13447 if (tp->t_port) {
13448 m->m_pkthdr.csum_flags = CSUM_UDP;
13449 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
13450 udp->uh_sum = in_pseudo(ip->ip_src.s_addr,
13451 ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP));
13452 th->th_sum = htons(0);
13455 csum_flags = m->m_pkthdr.csum_flags = CSUM_TCP;
13456 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
13457 th->th_sum = in_pseudo(ip->ip_src.s_addr,
13458 ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) +
13462 KASSERT(ip->ip_v == IPVERSION,
13463 ("%s: IP version incorrect: %d", __func__, ip->ip_v));
13475 m->m_pkthdr.csum_flags |= CSUM_TSO;
13477 m->m_pkthdr.tso_segsz = maxseg;
13494 log.u_bbr.flex1 = bbr->r_ctl.rc_hptsi_agg_delay;
13495 log.u_bbr.flex2 = (bbr->r_recovery_bw << 3);
13498 log.u_bbr.flex5 = bbr->rc_past_init_win;
13500 log.u_bbr.flex5 |= bbr->rc_no_pacing;
13502 log.u_bbr.flex5 |= tp->t_maxseg;
13503 log.u_bbr.flex6 = bbr->r_ctl.rc_pace_max_segs;
13504 log.u_bbr.flex7 = (bbr->rc_bbr_state << 8) | bbr_state_val(bbr);
13506 log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg;
13515 lgb = tcp_log_event(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_OUT, ERRNO_UNK,
13526 * m->m_pkthdr.len should have been set before cksum calcuration,
13537 ip6->ip6_hlim = in6_selecthlim(inp, NULL);
13544 ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6));
13547 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
13549 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
13551 if (tp->t_state == TCPS_SYN_SENT)
13556 error = ip6_output(m, inp->in6p_outputopts,
13557 &inp->inp_route6,
13561 if (error == EMSGSIZE && inp->inp_route6.ro_nh != NULL)
13562 mtu = inp->inp_route6.ro_nh->nh_mtu;
13570 ip->ip_len = htons(m->m_pkthdr.len);
13573 ip->ip_ttl = in6_selecthlim(inp, NULL);
13585 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) {
13586 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
13587 if (tp->t_port == 0 || len < V_tcp_minmss) {
13588 ip->ip_off |= htons(IP_DF);
13591 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
13594 if (tp->t_state == TCPS_SYN_SENT)
13599 error = ip_output(m, inp->inp_options, &inp->inp_route,
13602 if (error == EMSGSIZE && inp->inp_route.ro_nh != NULL)
13603 mtu = inp->inp_route.ro_nh->nh_mtu;
13607 lgb->tlb_errno = error;
13618 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
13619 (tp->t_flags & TF_SACK_PERMIT) &&
13620 tp->rcv_numsacks > 0)
13623 bbr->output_error_seen = 0;
13624 bbr->oerror_cnt = 0;
13625 bbr->bbr_segs_rcvd = 0;
13631 if (tp->snd_una == tp->snd_max) {
13638 bbr->r_ctl.rc_del_time = cts;
13643 counter_u64_add(bbr_out_size[(TCP_MSS_ACCT_ATIMER - 1)], 1);
13648 idx = len / (bbr_hptsi_bytes_min - bbr->rc_last_options);
13650 idx = (TCP_MSS_SMALL_MAX_SIZE_DIV - 1);
13664 if (tp->snd_una == tp->snd_max)
13665 bbr->r_ctl.rc_tlp_rxt_last_time = cts;
13677 if (bbr->rc_in_persist == 0) {
13685 if (tp->snd_una == tp->snd_max &&
13692 bbr->rc_tp->t_acktime = ticks;
13700 tp->snd_max = tp->iss + 1;
13702 if ((flags & TH_FIN) && ((tp->t_flags & TF_SENTFIN) == 0)) {
13703 tp->snd_max++;
13704 tp->t_flags |= TF_SENTFIN;
13708 tp->snd_max += len;
13721 if ((flags & TH_FIN) && ((tp->t_flags & TF_SENTFIN) == 0)) {
13723 tp->t_flags |= TF_SENTFIN;
13725 if (xlen && (tp->snd_una == tp->snd_max)) {
13731 bbr->rc_tp->t_acktime = ticks;
13734 tp->snd_max += xlen;
13741 * case of ENOBUFS we will fall out and become ack-clocked.
13749 bbr->r_ctl.rc_hptsi_agg_delay = 0;
13750 bbr->r_ctl.rc_agg_early = 0;
13751 bbr->r_agg_early_set = 0;
13752 bbr->output_error_seen = 1;
13753 if (bbr->oerror_cnt < 0xf)
13754 bbr->oerror_cnt++;
13755 if (bbr_max_net_error_cnt && (bbr->oerror_cnt >= bbr_max_net_error_cnt)) {
13757 return (-ENETDOWN);
13764 * slam him below a T-O (1MSS).
13766 if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {
13767 tp->snd_cwnd = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
13768 bbr->r_ctl.rc_lost_bytes)) - maxseg;
13769 if (tp->snd_cwnd < maxseg)
13770 tp->snd_cwnd = maxseg;
13772 slot = (bbr_error_base_paceout + 1) << bbr->oerror_cnt;
13774 if (bbr->bbr_hdrw_pacing)
13799 old_maxseg = tp->t_maxseg;
13803 tcp_mss_update(tp, -1, mtu, NULL, NULL);
13804 if (old_maxseg <= tp->t_maxseg) {
13806 tp->t_maxseg = old_maxseg - 40;
13807 if (tp->t_maxseg < V_tcp_mssdflt) {
13813 tp->t_flags2 |= TF2_PROC_SACK_PROHIBIT;
13815 tp->t_flags2 &= ~TF2_PROC_SACK_PROHIBIT;
13823 if ((tot_len + len) && (len >= tp->t_maxseg)) {
13825 bbr->r_ctl.rc_bbr_hptsi_gain,
13828 slot = (bbr_error_base_paceout + 2) << bbr->oerror_cnt;
13830 slot = (bbr_error_base_paceout + 2) << bbr->oerror_cnt;
13831 bbr->rc_output_starts_timer = 1;
13838 tp->t_softerror = error;
13844 if (TCPS_HAVERCVDSYN(tp->t_state)) {
13845 tp->t_softerror = error;
13850 slot = (bbr_error_base_paceout + 3) << bbr->oerror_cnt;
13851 bbr->rc_output_starts_timer = 1;
13856 } else if (((tp->t_flags & TF_GPUTINPROG) == 0) &&
13859 (bbr->rc_in_persist == 0)) {
13860 tp->gput_seq = bbr_seq;
13861 tp->gput_ack = bbr_seq +
13862 min(sbavail(&so->so_snd) - sb_offset, sendwin);
13863 tp->gput_ts = cts;
13864 tp->t_flags |= TF_GPUTINPROG;
13868 if ((bbr->bbr_hdw_pace_ena) &&
13869 (bbr->bbr_attempt_hdwr_pace == 0) &&
13870 (bbr->rc_past_init_win) &&
13871 (bbr->rc_bbr_state != BBR_STATE_STARTUP) &&
13872 (get_filter_value(&bbr->r_ctl.rc_delrate)) &&
13873 (inp->inp_route.ro_nh &&
13874 inp->inp_route.ro_nh->nh_ifp)) {
13886 bbr->bbr_attempt_hdwr_pace = 1;
13887 bbr->r_ctl.crte = tcp_set_pacing_rate(bbr->rc_tp,
13888 inp->inp_route.ro_nh->nh_ifp,
13892 if (bbr->r_ctl.crte) {
13894 bbr->r_ctl.crte->ptbl->rs_ifp,
13896 bbr->r_ctl.crte->rate,
13899 counter_u64_add(bbr_flows_nohdwr_pacing, -1);
13901 bbr->bbr_hdrw_pacing = 1;
13903 if (bbr->r_ctl.crte->rate < rate_wanted) {
13906 bbr->r_ctl.crte->rate, rate_wanted);
13909 bbr->gain_is_limited = 0;
13910 bbr->skip_gain = 0;
13915 inp->inp_route.ro_nh->nh_ifp,
13922 if (bbr->bbr_hdrw_pacing) {
13929 if (inp->inp_snd_tag == NULL) {
13931 bbr->bbr_hdrw_pacing = 0;
13932 } else if ((inp->inp_route.ro_nh == NULL) ||
13933 (inp->inp_route.ro_nh->nh_ifp != inp->inp_snd_tag->ifp)) {
13937 * and setup to re-attempt next go
13940 bbr->bbr_hdrw_pacing = 0;
13941 bbr->bbr_attempt_hdwr_pace = 0;
13942 tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp);
13951 if (SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv))
13952 tp->rcv_adv = tp->rcv_nxt + recwin;
13954 tp->last_ack_sent = tp->rcv_nxt;
13956 (bbr->r_ctl.rc_pace_max_segs > tp->t_maxseg) &&
13962 (IN_RECOVERY(tp->t_flags) == 0) &&
13963 (bbr->rc_in_persist == 0) &&
13964 (tot_len < bbr->r_ctl.rc_pace_max_segs)) {
13966 * For non-tso we need to goto again until we have sent out
13970 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
13972 tp->snd_nxt = tp->snd_max;
13980 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
13990 * Calculate/Re-Calculate the hptsi slot in usecs based on
13993 slot = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0);
13994 if (bbr->rc_no_pacing)
13997 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
13999 if (bbr->rc_use_google == 0)
14001 bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
14002 bbr->r_ctl.rc_lost_bytes)));
14003 bbr->rc_output_starts_timer = 1;
14004 if (bbr->bbr_use_rack_cheat &&
14006 ((bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts)) != NULL))) {
14011 if (bbr->bbr_hdrw_pacing && (bbr->hw_pacing_set == 0)) {
14017 bbr->r_ctl.bbr_hdwr_cnt_noset_snt++;
14018 if (bbr->r_ctl.bbr_hdwr_cnt_noset_snt >= bbr_hdwr_pacing_delay_cnt) {
14019 bbr->hw_pacing_set = 1;
14024 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
14026 tp->snd_nxt = tp->snd_max;
14061 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
14062 maxseg = tp->t_maxseg - bbr->rc_last_options;
14063 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
14064 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
14066 if (rsm->r_flags & BBR_ACKED)
14068 if ((rsm->r_end - rsm->r_start) > maxseg) {
14070 * We mark sack-passed on all the previous large
14073 rsm->r_flags |= BBR_SACK_PASSED;
14074 if (((rsm->r_flags & BBR_MARKED_LOST) == 0) &&
14075 bbr_is_lost(bbr, rsm, bbr->r_ctl.rc_rcvtime)) {
14076 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;
14077 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;
14078 rsm->r_flags |= BBR_MARKED_LOST;
14085 bbr->r_ctl.rc_resend = frsm;
14117 tp->t_flags2 |= TF2_CANNOT_DO_ECN;
14118 tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
14120 if (tp->t_in_hpts > IHPTS_NONE) {
14123 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
14125 if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
14126 if (TSTMP_GT(bbr->rc_pacer_started, cts)) {
14127 toval = bbr->rc_pacer_started - cts;
14132 } else if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
14133 if (TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) {
14134 toval = bbr->r_ctl.rc_timer_exp - cts;
14166 * socket option arguments. When it re-acquires the lock after the copy, it
14178 switch (sopt->sopt_level) {
14184 switch (sopt->sopt_name) {
14233 if (inp->inp_flags & INP_DROPPED) {
14237 if (tp->t_fb != &__tcp_bbr) {
14241 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
14242 switch (sopt->sopt_name) {
14245 bbr->r_ctl.bbr_hptsi_per_second = optval;
14249 bbr->r_ctl.bbr_hptsi_segments_delay_tar = optval;
14253 bbr->r_ctl.bbr_hptsi_segments_max = optval;
14257 bbr->r_ctl.bbr_hptsi_bytes_min = optval;
14261 bbr->r_ctl.bbr_cross_over = optval;
14265 if (optval && (bbr->rc_use_google == 0)) {
14266 /* Turn on the google mode */
14273 bbr->r_ctl.bbr_google_discount = optval;
14275 } else if ((optval == 0) && (bbr->rc_use_google == 1)) {
14276 /* Turn off the google mode */
14283 bbr->rc_use_ts_limit = 1;
14285 bbr->rc_use_ts_limit = 0;
14295 bbr->rc_init_win = optval;
14297 if ((bbr->rc_past_init_win == 0) && (twin > tp->snd_cwnd))
14298 tp->snd_cwnd = twin;
14307 bbr->r_ctl.rc_startup_pg = optval;
14308 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
14309 bbr->r_ctl.rc_bbr_hptsi_gain = optval;
14317 bbr->r_ctl.rc_drain_pg = optval;
14324 reset_time_small(&bbr->r_ctl.rc_rttprop, (optval * USECS_IN_SECOND));
14331 bbr->r_ctl.bbr_rttprobe_gain_val = optval;
14338 bbr->r_ctl.rc_probertt_int = optval;
14345 bbr->no_pacing_until = 0;
14346 bbr->rc_no_pacing = 0;
14348 bbr->no_pacing_until = optval;
14349 if ((bbr->r_ctl.rc_pkt_epoch < bbr->no_pacing_until) &&
14350 (bbr->rc_bbr_state == BBR_STATE_STARTUP)){
14352 bbr->rc_no_pacing = 1;
14359 bbr->rc_loss_exit = optval;
14366 bbr->r_ctl.rc_min_rto_ms = optval;
14370 bbr->rc_max_rto_sec = optval;
14373 /* Minimum time between rack t-o's in ms */
14375 bbr->r_ctl.rc_min_to = optval;
14381 bbr->r_ctl.rc_reorder_shift = optval;
14388 bbr->r_ctl.rc_reorder_fade = optval;
14394 bbr->rc_tlp_threshold = optval;
14400 if (bbr->rc_use_google) {
14406 bbr->bbr_use_rack_cheat = 1;
14408 bbr->bbr_use_rack_cheat = 0;
14413 bbr->r_ctl.bbr_hptsi_segments_floor = optval;
14420 bbr->r_ctl.bbr_utter_max = optval;
14428 bbr->rc_use_idle_restart = 1;
14430 bbr->rc_use_idle_restart = 0;
14435 bbr->bbr_init_win_cheat = 1;
14436 if (bbr->rc_past_init_win == 0) {
14438 cts = tcp_get_usecs(&bbr->rc_tv);
14442 bbr->bbr_init_win_cheat = 0;
14447 bbr->bbr_hdw_pace_ena = 1;
14448 bbr->bbr_attempt_hdwr_pace = 0;
14450 bbr->bbr_hdw_pace_ena = 0;
14452 if (bbr->r_ctl.crte != NULL) {
14453 tcp_rel_pacing_rate(bbr->r_ctl.crte, tp);
14454 bbr->r_ctl.crte = NULL;
14464 tp->t_delayed_ack = 0;
14466 tp->t_delayed_ack = 2;
14468 tp->t_delayed_ack = optval;
14469 if (tp->t_flags & TF_DELACK) {
14470 tp->t_flags &= ~TF_DELACK;
14471 tp->t_flags |= TF_ACKNOW;
14480 /* RACK added ms i.e. rack-rtt + reord + N */
14482 bbr->r_ctl.rc_pkt_delay = optval;
14488 bbr->rc_resends_use_tso = 1;
14490 bbr->rc_resends_use_tso = 0;
14495 bbr->rc_allow_data_af_clo = 1;
14497 bbr->rc_allow_data_af_clo = 0;
14501 if (bbr->rc_use_google == 0)
14504 bbr->r_use_policer = 1;
14506 bbr->r_use_policer = 0;
14512 bbr->ts_can_raise = 1;
14514 bbr->ts_can_raise = 0;
14518 if (bbr->rc_use_google) {
14522 bbr->r_ctl.rc_incr_tmrs = 1;
14524 bbr->r_ctl.rc_incr_tmrs = 0;
14529 if (bbr->rc_use_google) {
14539 bbr->r_ctl.rc_inc_tcp_oh = 1;
14541 bbr->r_ctl.rc_inc_tcp_oh = 0;
14543 bbr->r_ctl.rc_inc_ip_oh = 1;
14545 bbr->r_ctl.rc_inc_ip_oh = 0;
14547 bbr->r_ctl.rc_inc_enet_oh = 1;
14549 bbr->r_ctl.rc_inc_enet_oh = 0;
14556 tcp_log_socket_option(tp, sopt->sopt_name, optval, error);
14562 * return 0 on success, error-num on failure
14572 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
14583 switch (sopt->sopt_name) {
14585 optval = bbr->r_ctl.bbr_hptsi_per_second;
14588 optval = bbr->r_ctl.bbr_hptsi_segments_delay_tar;
14591 optval = bbr->r_ctl.bbr_hptsi_segments_max;
14594 optval = bbr->no_pacing_until;
14597 optval = bbr->r_ctl.bbr_hptsi_bytes_min;
14600 optval = bbr->r_ctl.bbr_cross_over;
14603 optval = bbr->rc_use_google;
14606 optval = bbr->rc_use_ts_limit;
14609 optval = bbr->rc_init_win;
14612 optval = bbr->r_ctl.rc_startup_pg;
14615 optval = bbr->r_ctl.rc_drain_pg;
14618 optval = bbr->r_ctl.rc_probertt_int;
14621 optval = (bbr->r_ctl.rc_rttprop.cur_time_limit / USECS_IN_SECOND);
14624 optval = bbr->r_ctl.bbr_rttprobe_gain_val;
14627 optval = bbr->rc_loss_exit;
14630 loptval = get_filter_value(&bbr->r_ctl.rc_delrate);
14633 optval = bbr->r_ctl.rc_min_rto_ms;
14636 optval = bbr->rc_max_rto_sec;
14640 optval = bbr->r_ctl.rc_pace_max_segs;
14643 /* Minimum time between rack t-o's in ms */
14644 optval = bbr->r_ctl.rc_min_to;
14648 optval = bbr->r_ctl.rc_reorder_shift;
14652 optval = bbr->r_ctl.rc_reorder_fade;
14656 optval = bbr->bbr_use_rack_cheat;
14659 optval = bbr->r_ctl.bbr_hptsi_segments_floor;
14662 optval = bbr->r_ctl.bbr_utter_max;
14666 optval = bbr->bbr_init_win_cheat;
14669 optval = bbr->rc_use_idle_restart;
14673 optval = bbr->rc_tlp_threshold;
14676 /* RACK added ms i.e. rack-rtt + reord + N */
14677 optval = bbr->r_ctl.rc_pkt_delay;
14680 optval = bbr->rc_resends_use_tso;
14683 optval = bbr->rc_allow_data_af_clo;
14686 optval = tp->t_delayed_ack;
14689 optval = bbr->bbr_hdw_pace_ena;
14692 optval = bbr->r_use_policer;
14695 optval = bbr->ts_can_raise;
14698 optval = bbr->r_ctl.rc_incr_tmrs;
14702 if (bbr->r_ctl.rc_inc_tcp_oh)
14704 if (bbr->r_ctl.rc_inc_ip_oh)
14706 if (bbr->r_ctl.rc_inc_enet_oh)
14714 if (sopt->sopt_name == TCP_BBR_USEDEL_RATE)
14722 * return 0 on success, error-num on failure
14727 if (sopt->sopt_dir == SOPT_SET) {
14729 } else if (sopt->sopt_dir == SOPT_GET) {
14732 panic("%s: sopt_dir $%d", __func__, sopt->sopt_dir);