Lines Matching defs:rsm
456 struct bbr_sendmap *rsm);
461 struct bbr_sendmap *rsm, uint32_t srtt, uint32_t cts);
564 struct bbr_sendmap *rsm;
574 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
575 if ((rsm == NULL) ||
585 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
586 if (rsm) {
587 idx = rsm->r_rtr_cnt - 1;
588 if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time))
589 tstmp_touse = rsm->r_tim_lastsent[idx];
615 if (rsm->r_flags & BBR_ACKED) {
616 rsm = bbr_find_lowest_rsm(bbr);
617 if (rsm == NULL) {
623 if (rsm->r_flags & BBR_SACK_PASSED) {
626 (rsm->r_flags & BBR_HAS_FIN)) {
634 thresh = bbr_calc_thresh_rack(bbr, srtt, cts, rsm);
635 idx = rsm->r_rtr_cnt - 1;
636 exp = rsm->r_tim_lastsent[idx] + thresh;
653 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext);
654 if (rsm == NULL) {
655 /* We found no rsm to TLP with. */
658 if (rsm->r_flags & BBR_HAS_FIN) {
660 rsm = NULL;
664 idx = rsm->r_rtr_cnt - 1;
665 if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time))
666 tstmp_touse = rsm->r_tim_lastsent[idx];
673 thresh = bbr_calc_thresh_tlp(tp, bbr, rsm, srtt, cts);
686 (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq)) {
694 if (rsm->r_start != bbr->r_ctl.rc_last_tlp_seq) {
700 bbr->r_ctl.rc_last_tlp_seq = rsm->r_start;
958 struct bbr_sendmap *rsm;
965 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
966 if (((rsm == NULL) || (tp->t_state < TCPS_ESTABLISHED)) &&
972 if (rsm == NULL) {
990 if (rsm && (rsm->r_flags & BBR_SACK_PASSED)) {
993 (rsm->r_flags & BBR_HAS_FIN)) {
1003 } else if (rsm && (tmr_up == PACE_TMR_RACK)) {
2502 bbr_log_thresh_choice(struct tcp_bbr *bbr, uint32_t cts, uint32_t thresh, uint32_t lro, uint32_t srtt, struct bbr_sendmap *rsm, uint8_t frm)
2511 log.u_bbr.flex4 = rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)];
2598 bbr_log_type_rsmclear(struct tcp_bbr *bbr, uint32_t cts, struct bbr_sendmap *rsm,
2606 log.u_bbr.flex2 = rsm->r_start;
2607 log.u_bbr.flex3 = rsm->r_end;
2608 log.u_bbr.flex4 = rsm->r_delivered;
2609 log.u_bbr.flex5 = rsm->r_rtr_cnt;
2610 log.u_bbr.flex6 = rsm->r_dupack;
2611 log.u_bbr.flex7 = rsm->r_tim_lastsent[0];
2612 log.u_bbr.flex8 = rsm->r_flags;
2893 bbr_isit_a_pkt_epoch(struct tcp_bbr *bbr, uint32_t cts, struct bbr_sendmap *rsm, int32_t line, int32_t cum_acked)
2895 if (SEQ_GEQ(rsm->r_delivered, bbr->r_ctl.rc_pkt_epoch_del)) {
3264 struct bbr_sendmap *rsm;
3267 rsm = uma_zalloc(bbr_zone, (M_NOWAIT | M_ZERO));
3268 if (rsm) {
3270 return (rsm);
3274 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);
3275 TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next);
3277 return (rsm);
3302 struct bbr_sendmap *rsm;
3318 rsm = bbr_alloc(bbr);
3319 if (rsm != NULL && limit_type) {
3320 rsm->r_limit_type = limit_type;
3323 return (rsm);
3327 bbr_free(struct tcp_bbr *bbr, struct bbr_sendmap *rsm)
3329 if (rsm->r_limit_type) {
3333 if (rsm->r_is_smallmap)
3335 if (bbr->r_ctl.rc_tlp_send == rsm)
3337 if (bbr->r_ctl.rc_resend == rsm) {
3340 if (bbr->r_ctl.rc_next == rsm)
3342 if (bbr->r_ctl.rc_sacklast == rsm)
3345 memset(rsm, 0, sizeof(struct bbr_sendmap));
3346 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next);
3347 rsm->r_limit_type = 0;
3352 uma_zfree(bbr_zone, rsm);
3907 bbr_cong_signal(struct tcpcb *tp, struct tcphdr *th, uint32_t type, struct bbr_sendmap *rsm)
3973 bbr_log_type_enter_rec(bbr, rsm->r_start);
4022 struct bbr_sendmap *rsm;
4025 * Walk the time-order transmitted list looking for an rsm that is
4029 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_tmap, r_tnext) {
4030 if (rsm->r_flags & BBR_ACKED) {
4036 return (rsm);
4040 bbr_find_high_nonack(struct tcp_bbr *bbr, struct bbr_sendmap *rsm)
4049 prsm = rsm;
4065 bbr_calc_thresh_rack(struct tcp_bbr *bbr, uint32_t srtt, uint32_t cts, struct bbr_sendmap *rsm)
4137 bbr_log_thresh_choice(bbr, cts, thresh, lro, srtt, rsm, BBR_TO_FRM_RACK);
4148 struct bbr_sendmap *rsm, uint32_t srtt,
4162 len = rsm->r_end - rsm->r_start;
4165 prsm = TAILQ_PREV(rsm, bbr_head, r_tnext);
4174 idx = rsm->r_rtr_cnt - 1;
4176 if (TSTMP_GEQ(rsm->r_tim_lastsent[nidx], prsm->r_tim_lastsent[idx])) {
4178 inter_gap = rsm->r_tim_lastsent[idx] - prsm->r_tim_lastsent[nidx];
4197 bbr_log_thresh_choice(bbr, cts, thresh, t_rxtcur, srtt, rsm, BBR_TO_FRM_TLP);
4265 bbr_is_lost(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t cts)
4270 cts, rsm);
4271 if ((cts - rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]) >= thresh) {
4290 struct bbr_sendmap *rsm;
4297 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
4298 if (rsm == NULL) {
4306 if (rsm->r_flags & BBR_ACKED) {
4311 rsm = bbr_find_lowest_rsm(bbr);
4312 if (rsm == NULL)
4315 idx = rsm->r_rtr_cnt - 1;
4316 if (SEQ_LEQ(cts, rsm->r_tim_lastsent[idx])) {
4321 if (bbr_is_lost(bbr, rsm, cts) &&
4322 ((rsm->r_dupack >= DUP_ACK_THRESHOLD) ||
4323 (rsm->r_flags & BBR_SACK_PASSED))) {
4324 if ((rsm->r_flags & BBR_MARKED_LOST) == 0) {
4325 rsm->r_flags |= BBR_MARKED_LOST;
4326 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;
4327 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;
4329 bbr_cong_signal(tp, NULL, CC_NDUPACK, rsm);
4331 if ((rsm->r_end - rsm->r_start) == 0)
4332 panic("tp:%p bbr:%p rsm:%p length is 0?", tp, bbr, rsm);
4334 return (rsm);
4381 bbr_clone_rsm(struct tcp_bbr *bbr, struct bbr_sendmap *nrsm, struct bbr_sendmap *rsm, uint32_t start)
4386 nrsm->r_end = rsm->r_end;
4387 nrsm->r_rtr_cnt = rsm->r_rtr_cnt;
4388 nrsm-> r_rtt_not_allowed = rsm->r_rtt_not_allowed;
4389 nrsm->r_flags = rsm->r_flags;
4393 rsm->r_flags &= ~BBR_HAS_FIN;
4394 nrsm->r_dupack = rsm->r_dupack;
4396 nrsm->r_is_gain = rsm->r_is_gain;
4397 nrsm->r_is_drain = rsm->r_is_drain;
4398 nrsm->r_delivered = rsm->r_delivered;
4399 nrsm->r_ts_valid = rsm->r_ts_valid;
4400 nrsm->r_del_ack_ts = rsm->r_del_ack_ts;
4401 nrsm->r_del_time = rsm->r_del_time;
4402 nrsm->r_app_limited = rsm->r_app_limited;
4403 nrsm->r_first_sent_time = rsm->r_first_sent_time;
4404 nrsm->r_flight_at_send = rsm->r_flight_at_send;
4406 nrsm->r_bbr_state = rsm->r_bbr_state;
4408 nrsm->r_tim_lastsent[idx] = rsm->r_tim_lastsent[idx];
4410 rsm->r_end = nrsm->r_start;
4414 if ((rsm->r_is_smallmap == 0) &&
4415 ((rsm->r_end - rsm->r_start) <= idx)) {
4417 rsm->r_is_smallmap = 1;
4539 struct bbr_sendmap *rsm = NULL;
4605 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);
4606 if (rsm && (rsm->r_flags & (BBR_ACKED | BBR_HAS_FIN))) {
4607 rsm = bbr_find_high_nonack(bbr, rsm);
4609 if (rsm == NULL) {
4617 TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
4618 if ((rsm->r_flags & BBR_RWND_COLLAPSED) == 0) {
4623 if (rsm == NULL) {
4625 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
4626 if (rsm == NULL)
4630 if ((rsm->r_end - rsm->r_start) > maxseg) {
4645 bbr_clone_rsm(bbr, nrsm, rsm, (rsm->r_end - maxseg));
4646 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
4647 if (rsm->r_in_tmap) {
4648 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
4651 rsm->r_flags &= (~BBR_HAS_FIN);
4652 rsm = nrsm;
4655 bbr->r_ctl.rc_tlp_send = rsm;
4657 if (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) {
4661 bbr->r_ctl.rc_last_tlp_seq = rsm->r_start;
4673 if (rsm)
4674 rsm->r_flags &= ~BBR_TLP;
4677 } else if (rsm) {
4678 rsm->r_flags |= BBR_TLP;
4680 if (rsm && (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) &&
4860 struct bbr_sendmap *rsm, *trsm = NULL;
4870 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
4871 if (rsm->r_flags & BBR_ACKED) {
4874 rsm->r_dupack = 0;
4875 if (rsm->r_in_tmap == 0) {
4878 TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
4880 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, trsm, rsm, r_tnext);
4882 rsm->r_in_tmap = 1;
4884 old_flags = rsm->r_flags;
4885 rsm->r_flags |= BBR_RXT_CLEARED;
4886 rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS);
4887 bbr_log_type_rsmclear(bbr, cts, rsm, old_flags, __LINE__);
4890 (rsm->r_start == tp->snd_una)) {
4898 if ((rsm->r_flags & BBR_MARKED_LOST) == 0) {
4899 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;
4900 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;
4907 rsm->r_flags |= BBR_SACK_PASSED | BBR_MARKED_LOST;
4908 rsm->r_flags &= ~BBR_WAS_SACKPASS;
4917 rsm->r_flags |= BBR_MARKED_LOST;
4918 rsm->r_flags &= ~BBR_WAS_SACKPASS;
4919 rsm->r_flags &= ~BBR_SACK_PASSED;
4922 trsm = rsm;
5290 struct bbr_sendmap *rsm;
5292 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
5293 if ((rsm == NULL) || (u_rsm == rsm))
5295 return(rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]);
5300 struct bbr_sendmap *rsm, uint32_t cts, uint32_t pacing_time)
5304 rsm->r_rtr_cnt++;
5305 rsm->r_dupack = 0;
5306 if (rsm->r_rtr_cnt > BBR_NUM_OF_RETRANS) {
5307 rsm->r_rtr_cnt = BBR_NUM_OF_RETRANS;
5308 rsm->r_flags |= BBR_OVERMAX;
5310 if (rsm->r_flags & BBR_RWND_COLLAPSED) {
5312 rsm->r_flags &= ~BBR_RWND_COLLAPSED;
5314 if (rsm->r_flags & BBR_MARKED_LOST) {
5316 rsm->r_flags &= ~BBR_MARKED_LOST;
5317 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
5319 if (rsm->r_flags & BBR_RXT_CLEARED) {
5328 rsm->r_flags &= ~BBR_RXT_CLEARED;
5330 if ((rsm->r_rtr_cnt > 1) && ((rsm->r_flags & BBR_TLP) == 0)) {
5331 bbr->r_ctl.rc_holes_rxt += (rsm->r_end - rsm->r_start);
5332 rsm->r_rtr_bytes += (rsm->r_end - rsm->r_start);
5334 idx = rsm->r_rtr_cnt - 1;
5335 rsm->r_tim_lastsent[idx] = cts;
5336 rsm->r_pacing_delay = pacing_time;
5337 rsm->r_delivered = bbr->r_ctl.rc_delivered;
5338 rsm->r_ts_valid = bbr->rc_ts_valid;
5340 rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts;
5342 rsm->r_app_limited = 1;
5344 rsm->r_app_limited = 0;
5346 rsm->r_bbr_state = bbr_state_val(bbr);
5348 rsm->r_bbr_state = 8;
5349 if (rsm->r_flags & BBR_ACKED) {
5353 old_flags = rsm->r_flags;
5354 rsm->r_flags &= ~BBR_ACKED;
5355 bbr_log_type_rsmclear(bbr, cts, rsm, old_flags, __LINE__);
5356 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
5360 if (rsm->r_in_tmap) {
5361 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
5363 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
5364 rsm->r_in_tmap = 1;
5365 if (rsm->r_flags & BBR_SACK_PASSED) {
5367 rsm->r_flags &= ~BBR_SACK_PASSED;
5368 rsm->r_flags |= BBR_WAS_SACKPASS;
5370 rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts);
5371 rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp,
5373 bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next);
5375 rsm->r_is_gain = 1;
5376 rsm->r_is_drain = 0;
5378 rsm->r_is_drain = 1;
5379 rsm->r_is_gain = 0;
5381 rsm->r_is_drain = 0;
5382 rsm->r_is_gain = 0;
5384 rsm->r_del_time = bbr->r_ctl.rc_del_time; /* TEMP GOOGLE CODE */
5395 struct bbr_sendmap *rsm, uint32_t cts, int32_t *lenp, uint32_t pacing_time)
5398 * We (re-)transmitted starting at rsm->r_start for some length
5406 c_end = rsm->r_start + len;
5407 if (SEQ_GEQ(c_end, rsm->r_end)) {
5410 * slopping into the next rsm.
5412 bbr_update_rsm(tp, bbr, rsm, cts, pacing_time);
5413 if (c_end == rsm->r_end) {
5420 act_len = rsm->r_end - rsm->r_start;
5422 return (rsm->r_end);
5436 * So here we are going to take the original rsm and make it what we
5442 bbr_clone_rsm(bbr, nrsm, rsm, c_end);
5443 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
5445 if (rsm->r_in_tmap) {
5446 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
5449 rsm->r_flags &= (~BBR_HAS_FIN);
5450 bbr_update_rsm(tp, bbr, rsm, cts, pacing_time);
5877 struct bbr_sendmap *rsm, *nrsm;
5946 rsm = bbr_alloc(bbr);
5947 if (rsm == NULL) {
5950 rsm->r_flags = 0;
5952 rsm->r_flags |= BBR_HAS_SYN;
5954 rsm->r_flags |= BBR_HAS_FIN;
5955 rsm->r_tim_lastsent[0] = cts;
5956 rsm->r_rtr_cnt = 1;
5957 rsm->r_rtr_bytes = 0;
5958 rsm->r_start = seq_out;
5959 rsm->r_end = rsm->r_start + len;
5960 rsm->r_dupack = 0;
5961 rsm->r_delivered = bbr->r_ctl.rc_delivered;
5962 rsm->r_pacing_delay = pacing_time;
5963 rsm->r_ts_valid = bbr->rc_ts_valid;
5965 rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts;
5966 rsm->r_del_time = bbr->r_ctl.rc_del_time;
5968 rsm->r_app_limited = 1;
5970 rsm->r_app_limited = 0;
5971 rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts);
5972 rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp,
5975 * Here we must also add in this rsm since snd_max
5978 rsm->r_flight_at_send += len;
5979 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next);
5980 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
5981 rsm->r_in_tmap = 1;
5983 rsm->r_bbr_state = bbr_state_val(bbr);
5985 rsm->r_bbr_state = 8;
5987 rsm->r_is_gain = 1;
5988 rsm->r_is_drain = 0;
5990 rsm->r_is_drain = 1;
5991 rsm->r_is_gain = 0;
5993 rsm->r_is_drain = 0;
5994 rsm->r_is_gain = 0;
6003 rsm = hintrsm;
6007 rsm = bbr->r_ctl.rc_next;
6010 rsm = NULL;
6012 if ((rsm) && (rsm->r_start == seq_out)) {
6017 seq_out = bbr_update_entry(tp, bbr, rsm, cts, &len, pacing_time);
6025 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
6026 if (rsm->r_start == seq_out) {
6027 seq_out = bbr_update_entry(tp, bbr, rsm, cts, &len, pacing_time);
6028 bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next);
6035 if (SEQ_GEQ(seq_out, rsm->r_start) && SEQ_LT(seq_out, rsm->r_end)) {
6043 bbr_update_rsm(tp, bbr, rsm, cts, pacing_time);
6047 * copy rsm to nrsm and then trim the front of rsm
6050 bbr_clone_rsm(bbr, nrsm, rsm, seq_out);
6051 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
6052 if (rsm->r_in_tmap) {
6053 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
6056 rsm->r_flags &= (~BBR_HAS_FIN);
6071 printf("seq_out:%u len:%d snd_una:%u snd_max:%u -- but rsm not found?\n",
6074 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
6075 printf("rsm:%p start:%u end:%u\n",
6076 rsm, rsm->r_start, rsm->r_end);
6120 * 2) Our timestamp indicating when we sent that packet (bbr->r_ctl.rsm->bbr_ts_check_our_cts).
6449 bbr_nf_measurement(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts)
6457 if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time))
6458 tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time);
6466 delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered);
6478 if (rsm->r_ts_valid &&
6483 ts_diff = max((bbr->r_ctl.last_inbound_ts - rsm->r_del_ack_ts), 1);
6491 rsm->r_del_ack_ts, 0,
6521 if (rsm->r_first_sent_time &&
6522 TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) {
6535 sbw = (uint64_t)(rsm->r_flight_at_send);
6537 sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time;
6538 sti += rsm->r_pacing_delay;
6546 rsm->r_first_sent_time, 0, (sbw >> 32),
6553 if ((rsm->r_app_limited == 0) ||
6557 0, 0, 0, 0, bbr->r_ctl.rc_del_time, rsm->r_del_time);
6563 bbr_google_measurement(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts)
6572 if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time))
6573 tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time);
6581 delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered);
6597 if (rsm->r_first_sent_time &&
6598 TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) {
6611 sbw = (uint64_t)(rsm->r_flight_at_send);
6613 sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time;
6614 sti += rsm->r_pacing_delay;
6622 rsm->r_first_sent_time, 0, (sbw >> 32),
6636 ((rsm->r_app_limited == 0) ||
6640 0, 0, 0, 0, bbr->r_ctl.rc_del_time, rsm->r_del_time);
6646 bbr_update_bbr_info(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts, uint32_t tsin,
6652 bbr->r_ctl.rc_delivered += (rsm->r_end - rsm->r_start);
6684 tcp_bbr_xmit_timer(bbr, rtt, rsm_send_time, rsm->r_start, tsin);
6704 bbr_log_type_bbrrttprop(bbr, rtt, rsm->r_end, uts, cts,
6705 match, rsm->r_start, rsm->r_flags);
6735 bbr_google_measurement(bbr, rsm, rtt, cts);
6737 bbr_nf_measurement(bbr, rsm, rtt, cts);
6761 struct bbr_sendmap *rsm, struct tcpopt *to, uint32_t cts, int32_t ack_type, uint32_t th_ack)
6766 if ((rsm->r_flags & BBR_ACKED) ||
6767 (rsm->r_flags & BBR_WAS_RENEGED) ||
6768 (rsm->r_flags & BBR_RXT_CLEARED)) {
6772 if (rsm->r_rtt_not_allowed) {
6776 if (rsm->r_rtr_cnt == 1) {
6780 if (TSTMP_GT(cts, rsm->r_tim_lastsent[0]))
6781 t = cts - rsm->r_tim_lastsent[0];
6785 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0,
6786 BBR_RTT_BY_EXACTMATCH, rsm->r_tim_lastsent[0], ack_type, to);
6799 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0,
6801 rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)],
6809 ((rsm->r_flags & BBR_OVERMAX) == 0)) {
6817 for (i = 0; i < rsm->r_rtr_cnt; i++) {
6818 if ((SEQ_GEQ(uts, (rsm->r_tim_lastsent[i] - fudge))) &&
6819 (SEQ_LEQ(uts, (rsm->r_tim_lastsent[i] + fudge)))) {
6820 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))
6821 t = cts - rsm->r_tim_lastsent[i];
6825 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_TSMATCHING,
6826 rsm->r_tim_lastsent[i], ack_type, to);
6827 if ((i + 1) < rsm->r_rtr_cnt) {
6830 } else if (rsm->r_flags & BBR_TLP) {
6848 i = rsm->r_rtr_cnt - 1;
6849 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))
6850 t = cts - rsm->r_tim_lastsent[i];
6865 if ((rsm->r_flags & BBR_OVERMAX) == 0) {
6867 if (rsm->r_rtr_cnt == 1)
6868 panic("rsm:%p bbr:%p rsm has overmax and only 1 retranmit flags:%x?", rsm, bbr, rsm->r_flags);
6870 i = rsm->r_rtr_cnt - 2;
6871 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))
6872 t = cts - rsm->r_tim_lastsent[i];
6875 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_EARLIER_RET,
6876 rsm->r_tim_lastsent[i], ack_type, to);
6884 bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts,
6892 if (rsm->r_flags & BBR_TLP)
6894 if ((rsm->r_flags & BBR_OVERMAX) == 0)
6895 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts,
6898 bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts,
6906 * Mark the SACK_PASSED flag on all entries prior to rsm send wise.
6910 struct tcp_bbr *bbr, struct bbr_sendmap *rsm)
6914 nrsm = rsm;
6917 if (nrsm == rsm) {
6955 struct bbr_sendmap *rsm, *nrsm;
6961 rsm = *prsm;
6962 if (rsm == NULL)
6966 if (rsm && SEQ_LT(start, rsm->r_start)) {
6968 TAILQ_FOREACH_REVERSE_FROM(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
6969 if (SEQ_GEQ(start, rsm->r_start) &&
6970 SEQ_LT(start, rsm->r_end)) {
6978 * Ok lets locate the block where this guy is fwd from rsm (if its
6981 TAILQ_FOREACH_FROM(rsm, &bbr->r_ctl.rc_map, r_next) {
6982 if (SEQ_GEQ(start, rsm->r_start) &&
6983 SEQ_LT(start, rsm->r_end)) {
6988 if (rsm == NULL) {
7006 rsm = nrsm;
7020 rsm = NULL;
7023 /* Ok we have an ACK for some piece of rsm */
7024 if (rsm->r_start != start) {
7028 if (bbr_sack_mergable(rsm, start, end))
7041 bbr_clone_rsm(bbr, nrsm, rsm, start);
7042 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
7043 if (rsm->r_in_tmap) {
7044 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
7047 rsm->r_flags &= (~BBR_HAS_FIN);
7048 rsm = nrsm;
7050 if (SEQ_GEQ(end, rsm->r_end)) {
7055 if ((rsm->r_flags & BBR_ACKED) == 0) {
7056 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_SACKED, 0);
7057 changed += (rsm->r_end - rsm->r_start);
7058 bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);
7059 bbr_log_sack_passed(tp, bbr, rsm);
7060 if (rsm->r_flags & BBR_MARKED_LOST) {
7061 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7064 if (rsm->r_flags & BBR_SACK_PASSED) {
7067 if (rsm->r_flags & BBR_MARKED_LOST) {
7068 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7074 rsm->r_flags |= BBR_ACKED;
7075 rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST);
7076 if (rsm->r_in_tmap) {
7077 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7078 rsm->r_in_tmap = 0;
7081 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_SACKED);
7082 if (end == rsm->r_end) {
7086 /* There is more not coverend by this rsm move on */
7087 start = rsm->r_end;
7088 nrsm = TAILQ_NEXT(rsm, r_next);
7089 rsm = nrsm;
7093 if (rsm->r_flags & BBR_ACKED) {
7098 if (bbr_sack_mergable(rsm, start, end))
7112 bbr_clone_rsm(bbr, nrsm, rsm, end);
7114 rsm->r_flags &= (~BBR_HAS_FIN);
7115 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
7116 if (rsm->r_in_tmap) {
7117 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
7121 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_SACKED, 0);
7122 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_SACKED);
7123 changed += (rsm->r_end - rsm->r_start);
7124 bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);
7125 bbr_log_sack_passed(tp, bbr, rsm);
7127 if (rsm->r_flags & BBR_MARKED_LOST) {
7128 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7130 if (rsm->r_flags & BBR_SACK_PASSED) {
7133 if (rsm->r_flags & BBR_MARKED_LOST) {
7134 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7140 rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST);
7141 rsm->r_flags |= BBR_ACKED;
7142 if (rsm->r_in_tmap) {
7143 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7144 rsm->r_in_tmap = 0;
7147 if (rsm && (rsm->r_flags & BBR_ACKED)) {
7153 nrsm = TAILQ_NEXT(rsm, r_next);
7157 rsm = bbr_merge_rsm(bbr, rsm, nrsm);
7160 nrsm = TAILQ_PREV(rsm, bbr_head, r_next);
7164 rsm = bbr_merge_rsm(bbr, nrsm, rsm);
7180 if (rsm)
7181 bbr->r_ctl.rc_sacklast = TAILQ_NEXT(rsm, r_next);
7184 *prsm = rsm;
7189 bbr_peer_reneges(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, tcp_seq th_ack)
7195 while (rsm && (rsm->r_flags & BBR_ACKED)) {
7198 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
7200 if (rsm->r_in_tmap) {
7201 panic("bbr:%p rsm:%p flags:0x%x in tmap?",
7202 bbr, rsm, rsm->r_flags);
7205 oflags = rsm->r_flags;
7206 if (rsm->r_flags & BBR_MARKED_LOST) {
7207 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7208 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7213 rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS | BBR_MARKED_LOST);
7214 rsm->r_flags |= BBR_WAS_RENEGED;
7215 rsm->r_flags |= BBR_RXT_CLEARED;
7216 bbr_log_type_rsmclear(bbr, bbr->r_ctl.rc_rcvtime, rsm, oflags, __LINE__);
7219 TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7220 tmap = rsm;
7222 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, tmap, rsm, r_tnext);
7223 tmap = rsm;
7234 rsm = TAILQ_NEXT(rsm, r_next);
7247 struct bbr_sendmap *rsm;
7252 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7253 if (rsm && (rsm->r_flags & BBR_HAS_SYN)) {
7254 if ((rsm->r_end - rsm->r_start) <= 1) {
7256 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
7257 rsm->r_rtr_bytes = 0;
7258 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);
7259 if (rsm->r_in_tmap) {
7260 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7261 rsm->r_in_tmap = 0;
7263 if (bbr->r_ctl.rc_next == rsm) {
7268 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_CUM_ACKED, 0);
7269 bbr_free(bbr, rsm);
7272 rsm->r_flags &= ~BBR_HAS_SYN;
7273 rsm->r_start++;
7289 struct bbr_sendmap *rsm;
7304 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7317 if (rsm && SEQ_GT(th_ack, rsm->r_start)) {
7318 changed = th_ack - rsm->r_start;
7319 } else if ((rsm == NULL) && ((th_ack - 1) == tp->iss)) {
7346 } else if (rsm == NULL) {
7357 if (rsm == NULL) {
7371 if (SEQ_LT(th_ack, rsm->r_start)) {
7375 rsm->r_start,
7381 } else if (th_ack == rsm->r_start) {
7390 rsm->r_dupack = 0;
7392 if (SEQ_GEQ(th_ack, rsm->r_end)) {
7396 if (rsm->r_flags & BBR_ACKED) {
7401 p_acked += (rsm->r_end - rsm->r_start);
7402 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
7406 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_CUM_ACKED, th_ack);
7407 if (rsm->r_flags & BBR_MARKED_LOST) {
7408 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7410 if (rsm->r_flags & BBR_SACK_PASSED) {
7418 if (rsm->r_flags & BBR_MARKED_LOST) {
7419 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7425 rsm->r_flags &= ~BBR_MARKED_LOST;
7427 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
7428 rsm->r_rtr_bytes = 0;
7429 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);
7430 if (rsm->r_in_tmap) {
7431 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7432 rsm->r_in_tmap = 0;
7434 if (bbr->r_ctl.rc_next == rsm) {
7438 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_CUM_ACKED);
7440 left = th_ack - rsm->r_end;
7442 bbr_free(bbr, rsm);
7444 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7449 if (rsm->r_flags & BBR_ACKED) {
7454 p_acked += (rsm->r_end - rsm->r_start);
7455 bbr->r_ctl.rc_sacked -= (th_ack - rsm->r_start);
7464 memcpy(&lrsm, rsm, sizeof(struct bbr_sendmap));
7468 if ((rsm->r_flags & BBR_MARKED_LOST) &&
7469 ((rsm->r_flags & BBR_ACKED) == 0)) {
7475 bbr->r_ctl.rc_lost_bytes -= th_ack - rsm->r_start;
7477 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_CUM_ACKED);
7478 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
7479 rsm->r_rtr_bytes = 0;
7481 rsm->r_start = th_ack;
7484 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7485 if (rsm && (rsm->r_flags & BBR_ACKED) && (th_ack == rsm->r_start)) {
7491 * us snd_una up to (rsm->r_end). We need to undo the acked
7495 * rsm->r_start in case we get an old ack where th_ack is
7498 bbr_peer_reneges(bbr, rsm, th->th_ack);
7504 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);
7505 if (rsm) {
7506 last_seq = rsm->r_end;
7608 rsm = bbr->r_ctl.rc_sacklast;
7610 acked = bbr_proc_sack_blk(tp, bbr, &sack_blocks[i], to, &rsm, cts);
7624 struct bbr_sendmap *rsm;
7626 rsm = bbr_check_recovery_mode(tp, bbr, cts);
7627 if (rsm) {
7636 bbr->r_ctl.rc_resend = rsm;
7661 struct bbr_sendmap *rsm;
7663 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
7664 if (rsm && (rsm->r_dupack < 0xff)) {
7665 rsm->r_dupack++;
7666 if (rsm->r_dupack >= DUP_ACK_THRESHOLD)
8016 struct bbr_sendmap *rsm, *nrsm;
8025 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
8027 if (rsm->r_flags & BBR_RWND_COLLAPSED)
8028 rsm->r_flags &= ~BBR_RWND_COLLAPSED;
8029 if (SEQ_GEQ(max_seq, rsm->r_start) &&
8030 SEQ_GEQ(rsm->r_end, max_seq)) {
8048 if ((max_seq != rsm->r_start) &&
8049 (max_seq != rsm->r_end)){
8053 res1 = max_seq - rsm->r_start;
8054 res2 = rsm->r_end - max_seq;
8064 /* Ok do we need to split this rsm? */
8065 if (max_seq == rsm->r_start) {
8067 nrsm = rsm;
8068 } else if (max_seq == rsm->r_end) {
8070 nrsm = TAILQ_NEXT(rsm, r_next);
8075 } else if (can_split && SEQ_LT(max_seq, rsm->r_end)) {
8080 nrsm = rsm;
8085 bbr_clone_rsm(bbr, nrsm, rsm, max_seq);
8086 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
8087 if (rsm->r_in_tmap) {
8088 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
8096 nrsm = rsm;
8113 struct bbr_sendmap *rsm;
8116 TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
8117 if (rsm->r_flags & BBR_RWND_COLLAPSED) {
8119 rsm->r_flags &= ~BBR_RWND_COLLAPSED;
10085 struct bbr_sendmap *rsm;
10087 rsm = bbr_alloc(bbr);
10088 if (rsm == NULL) {
10093 rsm->r_rtt_not_allowed = 1;
10094 rsm->r_tim_lastsent[0] = cts;
10095 rsm->r_rtr_cnt = 1;
10096 rsm->r_rtr_bytes = 0;
10097 rsm->r_start = tp->snd_una;
10098 rsm->r_end = tp->snd_max;
10099 rsm->r_dupack = 0;
10100 rsm->r_delivered = bbr->r_ctl.rc_delivered;
10101 rsm->r_ts_valid = 0;
10102 rsm->r_del_ack_ts = tp->ts_recent;
10103 rsm->r_del_time = cts;
10105 rsm->r_app_limited = 1;
10107 rsm->r_app_limited = 0;
10108 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next);
10109 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
10110 rsm->r_in_tmap = 1;
10112 rsm->r_bbr_state = bbr_state_val(bbr);
10114 rsm->r_bbr_state = 8;
10199 struct bbr_sendmap *rsm;
10214 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
10215 while (rsm) {
10216 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);
10217 uma_zfree(bbr_zone, rsm);
10218 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
10220 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);
10221 while (rsm) {
10222 TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next);
10223 uma_zfree(bbr_zone, rsm);
10224 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);
11287 struct bbr_sendmap *rsm;
11477 if (SEQ_GT(th->th_ack, tp->snd_una) && (rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map)) != NULL)
11478 kern_prefetch(rsm, &prev_state);
11687 bbr_do_send_accounting(struct tcpcb *tp, struct tcp_bbr *bbr, struct bbr_sendmap *rsm, int32_t len, int32_t error)
11692 if (rsm) {
11693 if (rsm->r_flags & BBR_TLP) {
11714 counter_u64_add(bbr_state_lost[rsm->r_bbr_state], len);
11858 struct bbr_sendmap *rsm = NULL;
12100 * use the rsm that needs to be sent so snd_nxt is ignored. At the
12116 rsm = NULL;
12124 rsm = bbr_alloc(bbr);
12125 if (rsm == NULL) {
12134 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next);
12136 rsm = NULL;
12150 rsm = bbr->r_ctl.rc_resend;
12155 rsm->r_flags &= ~BBR_TLP;
12157 if (SEQ_LT(rsm->r_start, tp->snd_una)) {
12159 panic("Huh, tp:%p bbr:%p rsm:%p start:%u < snd_una:%u\n",
12160 tp, bbr, rsm, rsm->r_start, tp->snd_una);
12164 rsm = NULL;
12168 if (rsm->r_flags & BBR_HAS_SYN) {
12173 rsm->r_start++;
12174 if (rsm->r_start == rsm->r_end) {
12179 rsm->r_flags &= ~BBR_HAS_SYN;
12180 len = rsm->r_end - rsm->r_start;
12184 rsm = NULL;
12189 len = rsm->r_end - rsm->r_start;
12195 sb_offset = rsm->r_start - tp->snd_una;
12203 rsm = NULL;
12212 rsm = bbr->r_ctl.rc_tlp_send;
12215 len = rsm->r_end - rsm->r_start;
12219 if (SEQ_GT(tp->snd_una, rsm->r_start)) {
12221 panic("tp:%p bbc:%p snd_una:%u rsm:%p r_start:%u",
12222 tp, bbr, tp->snd_una, rsm, rsm->r_start);
12225 rsm = NULL;
12229 sb_offset = rsm->r_start - tp->snd_una;
12236 if ((rsm == NULL) &&
12247 if (rsm && SEQ_LT(rsm->r_start, tp->snd_una)) {
12248 panic("tp:%p bbr:%p rsm:%p sb_offset:%u len:%u",
12249 tp, bbr, rsm, sb_offset, len);
12256 if (tp->t_flags & TF_NEEDFIN && (rsm == NULL))
12261 if (rsm && (rsm->r_flags & BBR_HAS_FIN)) {
12272 } else if (rsm) {
12348 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
12349 if (rsm) {
12350 len = rsm->r_end - rsm->r_start;
12351 if (rsm->r_flags & BBR_HAS_FIN)
12366 sb_offset = rsm->r_start - tp->snd_una;
12390 if ((flags & TH_SYN) && (rsm == NULL) &&
12404 } else if ((flags & TH_SYN) && rsm) {
12436 rsm = NULL;
12468 } else if ((rsm == NULL) &&
12531 (rsm == NULL) &&
12614 if (rsm)
12782 if (rsm == NULL) {
12803 (((rsm->r_flags & BBR_HAS_FIN) == 0) &&
13045 if (SEQ_LT(rsm->r_start, tp->snd_una)) {
13047 rsm, tp, bbr, rsm->r_start, tp->snd_una);
13072 if ((rsm == NULL) && len > pace_max_segs)
13100 panic("tp:%p bbr:%p len:%u sb_offset:%u sbavail:%u rsm:%p %u:%u:%u",
13101 tp, bbr, len, sb_offset, sbavail(sb), rsm,
13117 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
13119 if (rsm == NULL) {
13124 if (rsm->r_start != tp->snd_una) {
13134 len = rsm->r_end - rsm->r_start;
13145 if (rsm == NULL)
13151 if (rsm)
13157 if (rsm) {
13158 panic("tp:%p bbr:%p len:%u moff:%u sbavail:%u rsm:%p snd_una:%u rsm_start:%u flg:%x %u:%u:%u sr:%d ",
13160 sbavail(sb), rsm,
13161 tp->snd_una, rsm->r_flags, rsm->r_start,
13175 ((rsm == NULL) ? hw_tls : 0)
13344 /* All retransmits use the rsm to guide the send */
13345 th->th_seq = htonl(rsm->r_start);
13346 bbr_seq = rsm->r_start;
13507 if (rsm || sack_rxmit) {
13558 ((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0),
13600 ((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0), 0,
13617 tcp_account_for_send(tp, len, (rsm != NULL), doing_tlp, hw_tls);
13629 if ((len > 0) && (rsm == NULL)) {
13658 * otherwise the state of the rsm could change and we account to the
13662 bbr_do_send_accounting(tp, bbr, rsm, len, error);
13669 cts, mb, &abandon, rsm, 0, sb);
13694 if (flags & (TH_SYN | TH_FIN) && (rsm == NULL)) {
13858 (rsm == NULL) &&
13974 if (rsm != NULL) {
13975 rsm = NULL;
13978 rsm = NULL;
14053 struct bbr_sendmap *rsm, *frsm = NULL;
14064 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
14066 if (rsm->r_flags & BBR_ACKED)
14068 if ((rsm->r_end - rsm->r_start) > maxseg) {
14073 rsm->r_flags |= BBR_SACK_PASSED;
14074 if (((rsm->r_flags & BBR_MARKED_LOST) == 0) &&
14075 bbr_is_lost(bbr, rsm, bbr->r_ctl.rc_rcvtime)) {
14076 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;
14077 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;
14078 rsm->r_flags |= BBR_MARKED_LOST;
14081 frsm = rsm;