H A D | bbr.c | 1 /*- 2 * Copyright (c) 2016-2020 Netflix, Inc. 29 * BBR - Congestion Based Congestion Control 159 /* Should the following be dynamic too -- loss wise */ 193 * num-measures > min(0) and 227 /* thresholds for reduction on drain in sub-states/drain */ 265 * means in non-recovery/retransmission scenarios 266 * cwnd will never be reached by the flight-size. 271 static int32_t bbr_sack_not_required = 0; /* set to one to allow non-sac 553 bbr_timer_start(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts) bbr_timer_start() argument 729 bbr_start_hpts_timer(struct tcp_bbr * bbr,struct tcpcb * tp,uint32_t cts,int32_t frm,int32_t slot,uint32_t tot_len) bbr_start_hpts_timer() argument 950 bbr_timer_audit(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts,struct sockbuf * sb) bbr_timer_audit() argument 1098 bbr_calc_time(uint32_t cts,uint32_t earlier_time) bbr_calc_time() argument 1857 bbr_fill_in_logging_data(struct tcp_bbr * bbr,struct tcp_log_bbr * l,uint32_t cts) bbr_fill_in_logging_data() argument 1924 bbr_log_type_just_return(struct tcp_bbr * bbr,uint32_t cts,uint32_t tlen,uint8_t hpts_calling,uint8_t reason,uint32_t p_maxseg,int len) bbr_log_type_just_return() argument 1968 bbr_log_msgsize_fail(struct tcp_bbr * bbr,struct tcpcb * tp,uint32_t len,uint32_t maxseg,uint32_t mtu,int32_t csum_flags,int32_t tso,uint32_t cts) bbr_log_msgsize_fail() argument 2009 bbr_log_pkt_epoch(struct tcp_bbr * bbr,uint32_t cts,uint32_t line,uint32_t lost,uint32_t del) bbr_log_pkt_epoch() argument 2034 bbr_log_time_epoch(struct tcp_bbr * bbr,uint32_t cts,uint32_t line,uint32_t epoch_time) bbr_log_time_epoch() argument 2077 bbr_log_type_statechange(struct tcp_bbr * bbr,uint32_t cts,int32_t line) bbr_log_type_statechange() argument 2104 bbr_log_rtt_shrinks(struct tcp_bbr * bbr,uint32_t cts,uint32_t applied,uint32_t rtt,uint32_t line,uint8_t reas,uint16_t cond) bbr_log_rtt_shrinks() argument 2197 bbr_log_type_pesist(struct tcp_bbr * bbr,uint32_t cts,uint32_t time_in,int32_t line,uint8_t enter_exit) bbr_log_type_pesist() argument 2214 bbr_log_ack_clear(struct tcp_bbr * bbr,uint32_t cts) bbr_log_ack_clear() argument 2235 bbr_log_ack_event(struct tcp_bbr * bbr,struct tcphdr * th,struct tcpopt * to,uint32_t tlen,uint16_t nsegs,uint32_t cts,int32_t nxt_pkt,struct mbuf * m) bbr_log_ack_event() argument 2283 bbr_log_doseg_done(struct tcp_bbr * bbr,uint32_t cts,int32_t nxt_pkt,int32_t did_out) bbr_log_doseg_done() argument 2307 bbr_log_enobuf_jmp(struct tcp_bbr * bbr,uint32_t len,uint32_t cts,int32_t line,uint32_t o_len,uint32_t segcnt,uint32_t segsiz) bbr_log_enobuf_jmp() argument 2327 bbr_log_to_processing(struct tcp_bbr * bbr,uint32_t cts,int32_t ret,int32_t timers,uint8_t hpts_calling) bbr_log_to_processing() argument 2349 bbr_log_to_event(struct tcp_bbr * bbr,uint32_t cts,int32_t to_num) bbr_log_to_event() argument 2377 bbr_log_startup_event(struct tcp_bbr * bbr,uint32_t cts,uint32_t flex1,uint32_t flex2,uint32_t flex3,uint8_t reason) bbr_log_startup_event() argument 2400 bbr_log_hpts_diag(struct tcp_bbr * bbr,uint32_t cts,struct hpts_diag * diag) bbr_log_hpts_diag() argument 2436 bbr_log_timer_var(struct tcp_bbr * bbr,int mode,uint32_t cts,uint32_t time_since_sent,uint32_t srtt,uint32_t thresh,uint32_t to) bbr_log_timer_var() argument 2460 bbr_log_pacing_delay_calc(struct tcp_bbr * bbr,uint16_t gain,uint32_t len,uint32_t cts,uint32_t usecs,uint64_t bw,uint32_t override,int mod) bbr_log_pacing_delay_calc() argument 2486 bbr_log_to_start(struct tcp_bbr * bbr,uint32_t cts,uint32_t to,int32_t slot,uint8_t which) bbr_log_to_start() argument 2510 bbr_log_thresh_choice(struct tcp_bbr * bbr,uint32_t cts,uint32_t thresh,uint32_t lro,uint32_t srtt,struct bbr_sendmap * rsm,uint8_t frm) bbr_log_thresh_choice() argument 2533 bbr_log_to_cancel(struct tcp_bbr * bbr,int32_t line,uint32_t cts,uint8_t hpts_removed) bbr_log_to_cancel() argument 2578 bbr_log_type_tsosize(struct tcp_bbr * bbr,uint32_t cts,uint32_t tsosz,uint32_t tls,uint32_t old_val,uint32_t maxseg,int hdwr) bbr_log_type_tsosize() argument 2606 bbr_log_type_rsmclear(struct tcp_bbr * bbr,uint32_t cts,struct bbr_sendmap * rsm,uint32_t flags,uint32_t line) bbr_log_type_rsmclear() argument 2632 bbr_log_type_bbrupd(struct tcp_bbr * bbr,uint8_t flex8,uint32_t cts,uint32_t flex3,uint32_t flex2,uint32_t flex5,uint32_t flex6,uint32_t pkts_out,int flex7,uint32_t flex4,uint32_t flex1) bbr_log_type_bbrupd() argument 2665 bbr_log_type_ltbw(struct tcp_bbr * bbr,uint32_t cts,int32_t reason,uint32_t newbw,uint32_t obw,uint32_t diff,uint32_t tim) bbr_log_type_ltbw() argument 2716 bbr_type_log_hdwr_pacing(struct tcp_bbr * bbr,const struct ifnet * ifp,uint64_t rate,uint64_t hw_rate,int line,uint32_t cts,int error) bbr_type_log_hdwr_pacing() argument 2746 bbr_log_type_bbrsnd(struct tcp_bbr * bbr,uint32_t len,uint32_t slot,uint32_t del_by,uint32_t cts,uint32_t line,uint32_t prev_delay) bbr_log_type_bbrsnd() argument 2769 bbr_log_type_bbrrttprop(struct tcp_bbr * bbr,uint32_t t,uint32_t end,uint32_t tsconv,uint32_t cts,int32_t match,uint32_t seq,uint8_t flags) bbr_log_type_bbrrttprop() argument 2792 bbr_log_exit_gain(struct tcp_bbr * bbr,uint32_t cts,int32_t entry_method) bbr_log_exit_gain() argument 2851 bbr_set_pktepoch(struct tcp_bbr * bbr,uint32_t cts,int32_t line) bbr_set_pktepoch() argument 2889 bbr_set_epoch(struct tcp_bbr * bbr,uint32_t cts,int32_t line) bbr_set_epoch() argument 2901 bbr_isit_a_pkt_epoch(struct tcp_bbr * bbr,uint32_t cts,struct bbr_sendmap * rsm,int32_t line,int32_t cum_acked) bbr_isit_a_pkt_epoch() argument 3014 bbr_reset_lt_bw_interval(struct tcp_bbr * bbr,uint32_t cts) bbr_reset_lt_bw_interval() argument 3023 bbr_reset_lt_bw_sampling(struct tcp_bbr * bbr,uint32_t cts) bbr_reset_lt_bw_sampling() argument 3032 bbr_lt_bw_samp_done(struct tcp_bbr * bbr,uint64_t bw,uint32_t cts,uint32_t timin) bbr_lt_bw_samp_done() argument 3086 bbr_pick_probebw_substate(struct tcp_bbr * bbr,uint32_t cts) bbr_pick_probebw_substate() argument 3114 bbr_lt_bw_sampling(struct tcp_bbr * bbr,uint32_t cts,int32_t loss_detected) bbr_lt_bw_sampling() argument 3540 bbr_get_pacing_delay(struct tcp_bbr * bbr,uint16_t gain,int32_t len,uint32_t cts,int nolog) bbr_get_pacing_delay() argument 3906 bbr_setup_red_bw(struct tcp_bbr * bbr,uint32_t cts) bbr_setup_red_bw() argument 4076 bbr_calc_thresh_rack(struct tcp_bbr * bbr,uint32_t srtt,uint32_t cts,struct bbr_sendmap * rsm) bbr_calc_thresh_rack() argument 4160 bbr_calc_thresh_tlp(struct tcpcb * tp,struct tcp_bbr * bbr,struct bbr_sendmap * rsm,uint32_t srtt,uint32_t cts) bbr_calc_thresh_tlp() argument 4276 bbr_is_lost(struct tcp_bbr * bbr,struct bbr_sendmap * rsm,uint32_t cts) bbr_is_lost() argument 4293 bbr_check_recovery_mode(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts) bbr_check_recovery_mode() argument 4358 bbr_timeout_rack(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts) bbr_timeout_rack() argument 4545 bbr_timeout_tlp(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts) bbr_timeout_tlp() argument 4713 bbr_timeout_delack(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts) bbr_timeout_delack() argument 4734 bbr_timeout_persist(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts) bbr_timeout_persist() argument 4812 bbr_timeout_keepalive(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts) bbr_timeout_keepalive() argument 4873 uint32_t cts, lost; bbr_remxt_tmr() local 4956 bbr_timeout_rxt(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts) bbr_timeout_rxt() argument 5187 bbr_process_timers(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts,uint8_t hpts_calling) bbr_process_timers() argument 5247 bbr_timer_cancel(struct tcp_bbr * bbr,int32_t line,uint32_t cts) bbr_timer_cancel() argument 5299 bbr_get_earliest_send_outstanding(struct tcp_bbr * bbr,struct bbr_sendmap * u_rsm,uint32_t cts) bbr_get_earliest_send_outstanding() argument 5311 bbr_update_rsm(struct tcpcb * tp,struct tcp_bbr * bbr,struct bbr_sendmap * rsm,uint32_t cts,uint32_t pacing_time) bbr_update_rsm() argument 5406 bbr_update_entry(struct tcpcb * tp,struct tcp_bbr * bbr,struct bbr_sendmap * rsm,uint32_t cts,int32_t * lenp,uint32_t pacing_time) bbr_update_entry() argument 5478 bbr_setup_less_of_rate(struct tcp_bbr * bbr,uint32_t cts,uint64_t act_rate,uint64_t rate_wanted) bbr_setup_less_of_rate() argument 5502 bbr_update_hardware_pacing_rate(struct tcp_bbr * bbr,uint32_t cts) bbr_update_hardware_pacing_rate() argument 5564 bbr_adjust_for_hw_pacing(struct tcp_bbr * bbr,uint32_t cts) bbr_adjust_for_hw_pacing() argument 5686 tcp_bbr_tso_size_check(struct tcp_bbr * bbr,uint32_t cts) tcp_bbr_tso_size_check() argument 5883 bbr_log_output(struct tcp_bbr * bbr,struct tcpcb * tp,struct tcpopt * to,int32_t len,uint32_t seq_out,uint16_t th_flags,int32_t err,uint32_t cts,struct mbuf * mb,int32_t * abandon,struct bbr_sendmap * hintrsm,uint32_t delay_calc,struct sockbuf * sb) bbr_log_output() argument 6255 tcp_bbr_xmit_timer_commit(struct tcp_bbr * bbr,struct tcpcb * tp,uint32_t cts) tcp_bbr_xmit_timer_commit() argument 6409 bbr_set_reduced_rtt(struct tcp_bbr * bbr,uint32_t cts,uint32_t line) bbr_set_reduced_rtt() argument 6425 tcp_bbr_commit_bw(struct tcp_bbr * bbr,uint32_t cts) tcp_bbr_commit_bw() argument 6462 bbr_nf_measurement(struct tcp_bbr * bbr,struct bbr_sendmap * rsm,uint32_t rtt,uint32_t cts) bbr_nf_measurement() argument 6576 bbr_google_measurement(struct tcp_bbr * bbr,struct bbr_sendmap * rsm,uint32_t rtt,uint32_t cts) bbr_google_measurement() argument 6659 bbr_update_bbr_info(struct tcp_bbr * bbr,struct bbr_sendmap * rsm,uint32_t rtt,uint32_t cts,uint32_t tsin,uint32_t uts,int32_t match,uint32_t rsm_send_time,int32_t ack_type,struct tcpopt * to) bbr_update_bbr_info() argument 6760 bbr_ts_convert(uint32_t cts) bbr_ts_convert() argument 6774 bbr_update_rtt(struct tcpcb * tp,struct tcp_bbr * bbr,struct bbr_sendmap * rsm,struct tcpopt * to,uint32_t cts,int32_t ack_type,uint32_t th_ack) bbr_update_rtt() argument 6968 bbr_proc_sack_blk(struct tcpcb * tp,struct tcp_bbr * bbr,struct sackblk * sack,struct tcpopt * to,struct bbr_sendmap ** prsm,uint32_t cts) bbr_proc_sack_blk() argument 7265 uint32_t cts; bbr_log_syn() local 7310 uint32_t cts, acked, ack_point, sack_changed = 0; bbr_log_ack() local 7861 bbr_enter_persist(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts,int32_t line) bbr_enter_persist() argument 7890 bbr_restart_after_idle(struct tcp_bbr * bbr,uint32_t cts,uint32_t idle_time) bbr_restart_after_idle() argument 7920 bbr_exit_persist(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts,int32_t line) bbr_exit_persist() argument 9923 uint32_t cts; bbr_init() local 10267 bbr_substate_change(struct tcp_bbr * bbr,uint32_t cts,int32_t line,int dolog) bbr_substate_change() argument 10412 bbr_set_probebw_google_gains(struct tcp_bbr * bbr,uint32_t cts,uint32_t losses) bbr_set_probebw_google_gains() argument 10450 bbr_set_probebw_gains(struct tcp_bbr * bbr,uint32_t cts,uint32_t losses) bbr_set_probebw_gains() argument 10643 bbr_enter_probe_rtt(struct tcp_bbr * bbr,uint32_t cts,int32_t line) bbr_enter_probe_rtt() argument 10717 bbr_check_probe_rtt_limits(struct tcp_bbr * bbr,uint32_t cts) bbr_check_probe_rtt_limits() argument 10787 bbr_exit_probe_rtt(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts) bbr_exit_probe_rtt() argument 10848 bbr_should_enter_probe_rtt(struct tcp_bbr * bbr,uint32_t cts) bbr_should_enter_probe_rtt() argument 10865 bbr_google_startup(struct tcp_bbr * bbr,uint32_t cts,int32_t pkt_epoch) bbr_google_startup() argument 10891 bbr_state_startup(struct tcp_bbr * bbr,uint32_t cts,int32_t epoch,int32_t pkt_epoch) bbr_state_startup() argument 11031 bbr_state_change(struct tcp_bbr * bbr,uint32_t cts,int32_t epoch,int32_t pkt_epoch,uint32_t losses) bbr_state_change() argument 11249 bbr_check_bbr_for_state(struct tcp_bbr * bbr,uint32_t cts,int32_t line,uint32_t losses) bbr_check_bbr_for_state() argument 11269 uint32_t cts, lcts; bbr_do_segment_nounlock() local 11637 bbr_what_can_we_send(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t sendwin,uint32_t avail,int32_t sb_offset,uint32_t cts) bbr_what_can_we_send() argument 11806 uint32_t cts; bbr_output_wtime() local 14098 uint32_t cts; bbr_switch_failed() local 14425 uint32_t cts; bbr_set_sockopt() local [all...] |
H A D | rack.c | 1 /*- 2 * Copyright (c) 2016-2020 Netflix, Inc. 162 * - Matt Mathis's Rate Halving which slowly drops 165 * - Yuchung Cheng's RACK TCP (for which its named) that 168 * - Reorder Detection of RFC4737 and the Tail-Loss probe draft 186 * TCP output is also over-written with a new version since it 191 static int32_t rack_tlp_limit = 2; /* No more than 2 TLPs w-out new data */ 194 static int32_t rack_reorder_fade = 60000000; /* 0 - never fade, def 60,000,000 195 * - 6 2794 rack_log_to_start(struct tcp_rack * rack,uint32_t cts,uint32_t to,int32_t slot,uint8_t which) rack_log_to_start() argument 3128 rack_log_type_bbrsnd(struct tcp_rack * rack,uint32_t len,uint32_t slot,uint32_t cts,struct timeval * tv,int line) rack_log_type_bbrsnd() argument 3158 rack_log_doseg_done(struct tcp_rack * rack,uint32_t cts,int32_t nxt_pkt,int32_t did_out,int way_out,int nsegs) rack_log_doseg_done() argument 3233 rack_log_type_just_return(struct tcp_rack * rack,uint32_t cts,uint32_t tlen,uint32_t slot,uint8_t hpts_calling,int reason,uint32_t cwnd_to_use) rack_log_type_just_return() argument 3340 rack_log_to_processing(struct tcp_rack * rack,uint32_t cts,int32_t ret,int32_t timers) rack_log_to_processing() argument 3908 rack_increase_bw_mul(struct tcp_rack * rack,int timely_says,uint64_t cur_bw,uint64_t last_bw_est,int override) rack_increase_bw_mul() argument 6491 rack_calc_thresh_rack(struct tcp_rack * rack,uint32_t srtt,uint32_t cts,int line,int log_allowed) rack_calc_thresh_rack() argument 6737 rack_timer_start(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts,int sup_rack) rack_timer_start() argument 6965 rack_enter_persist(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts,tcp_seq snd_una) rack_enter_persist() argument 7011 rack_exit_persist(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts) rack_exit_persist() argument 7075 rack_log_hpts_diag(struct tcp_rack * rack,uint32_t cts,struct hpts_diag * diag,struct timeval * tv) rack_log_hpts_diag() argument 7135 rack_start_hpts_timer(struct tcp_rack * rack,struct tcpcb * tp,uint32_t cts,int32_t slot,uint32_t tot_len_this_send,int sup_rack) rack_start_hpts_timer() argument 7448 rack_mark_lost(struct tcpcb * tp,struct tcp_rack * rack,struct rack_sendmap * rsm,uint32_t cts) rack_mark_lost() argument 7482 rack_timeout_rack(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts) rack_timeout_rack() argument 7764 rack_timeout_tlp(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts,uint8_t * doing_tlp) rack_timeout_tlp() argument 7953 rack_timeout_delack(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts) rack_timeout_delack() argument 7999 rack_timeout_persist(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts) rack_timeout_persist() argument 8072 rack_timeout_keepalive(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts) rack_timeout_keepalive() argument 8259 rack_timeout_rxt(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts) rack_timeout_rxt() argument 8545 rack_process_timers(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts,uint8_t hpts_calling,uint8_t * doing_tlp) rack_process_timers() argument 8657 rack_timer_cancel(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts,int line) rack_timer_cancel() argument 8962 rack_log_output(struct tcpcb * tp,struct tcpopt * to,int32_t len,uint32_t seq_out,uint16_t th_flags,int32_t err,uint64_t cts,struct rack_sendmap * hintrsm,uint32_t add_flag,struct mbuf * s_mb,uint32_t s_moff,int hw_tls,int segsiz) rack_log_output() argument 9539 rack_update_rtt(struct tcpcb * tp,struct tcp_rack * rack,struct rack_sendmap * rsm,struct tcpopt * to,uint32_t cts,int32_t ack_type,tcp_seq th_ack) rack_update_rtt() argument 9787 rack_log_sack_passed(struct tcpcb * tp,struct tcp_rack * rack,struct rack_sendmap * rsm,uint32_t cts) rack_log_sack_passed() argument 10013 rack_proc_sack_blk(struct tcpcb * tp,struct tcp_rack * rack,struct sackblk * sack,struct tcpopt * to,struct rack_sendmap ** prsm,uint32_t cts,uint32_t segsiz) rack_proc_sack_blk() argument 10905 rack_process_to_cumack(struct tcpcb * tp,struct tcp_rack * rack,register uint32_t th_ack,uint32_t cts,struct tcpopt * to,uint64_t acktime) rack_process_to_cumack() argument 11490 uint32_t cts, acked, ack_point; rack_log_ack() local 11783 uint32_t cts; rack_strike_dupack() local 12014 rack_set_dgp_hybrid_mode(struct tcp_rack * rack,tcp_seq seq,uint32_t len,uint64_t cts) rack_set_dgp_hybrid_mode() argument 12187 rack_chk_req_and_hybrid_on_out(struct tcp_rack * rack,tcp_seq seq,uint32_t len,uint64_t cts) rack_chk_req_and_hybrid_on_out() argument 13317 rack_fastack(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t nxt_pkt,uint32_t cts) rack_fastack() argument 15058 uint32_t cts; rack_switch_failed() local 16054 rack_do_win_updates(struct tcpcb * tp,struct tcp_rack * rack,uint32_t tiwin,uint32_t seq,uint32_t ack,uint32_t cts) rack_do_win_updates() argument 16458 uint32_t tiwin, ms_cts, cts, acked, acked_amount, high_seq, win_seq, the_win, win_upd_ack; rack_do_compressed_ack_processing() local 17127 uint32_t cts, us_cts, ms_cts; rack_do_segment_nounlock() local 19257 rack_log_queue_level(struct tcpcb * tp,struct tcp_rack * rack,int len,struct timeval * tv,uint32_t cts) rack_log_queue_level() argument 19286 rack_check_queue_level(struct tcp_rack * rack,struct tcpcb * tp,struct timeval * tv,uint32_t cts,int len,uint32_t segsiz) rack_check_queue_level() argument 19354 rack_fast_rsm_output(struct tcpcb * tp,struct tcp_rack * rack,struct rack_sendmap * rsm,uint64_t ts_val,uint32_t cts,uint32_t ms_cts,struct timeval * tv,int len,uint8_t doing_tlp) rack_fast_rsm_output() argument 19942 rack_fast_output(struct tcpcb * tp,struct tcp_rack * rack,uint64_t ts_val,uint32_t cts,uint32_t ms_cts,struct timeval * tv,long tot_len,int * send_err) rack_fast_output() argument 20494 rack_check_collapsed(struct tcp_rack * rack,uint32_t cts) rack_check_collapsed() argument 20632 uint32_t cts, ms_cts, delayed, early; rack_output() local 25210 uint32_t cts; rack_wake_check() local [all...] |