Lines Matching +full:cts +full:- +full:override
1 /*-
2 * Copyright (c) 2016-2018 Netflix, Inc.
3 * Copyright (c) 2016-2021 Mellanox Technologies.
77 ae->timestamp = m->m_pkthdr.rcv_tstmp;
78 ae->flags = 0;
79 if (m->m_flags & M_TSTMP_LRO)
80 ae->flags |= TSTMP_LRO;
81 else if (m->m_flags & M_TSTMP)
82 ae->flags |= TSTMP_HDWR;
83 ae->seq = th->th_seq;
84 ae->ack = th->th_ack;
85 ae->flags |= tcp_get_flags(th);
87 ae->ts_value = ntohl(ts_ptr[1]);
88 ae->ts_echo = ntohl(ts_ptr[2]);
89 ae->flags |= HAS_TSTMP;
91 ae->win = th->th_win;
92 ae->codepoint = iptos;
100 * a) Is what is present capable of being ack-compressed,
101 * we can ack-compress if there is no options or just
109 switch (th->th_off << 2) {
135 if (m->m_pkthdr.lro_tcp_d_len)
148 if (tp->t_fb->tfb_early_wake_check != NULL)
149 return ((tp->t_fb->tfb_early_wake_check)(tp));
163 uint32_t cts;
165 cts = tcp_get_usecs(&tv);
170 log.u_bbr.flex2 = m->m_pkthdr.len;
173 if (le->m_head) {
174 log.u_bbr.flex3 = le->m_head->m_pkthdr.lro_nsegs;
175 log.u_bbr.flex4 = le->m_head->m_pkthdr.lro_tcp_d_len;
176 log.u_bbr.flex5 = le->m_head->m_pkthdr.len;
177 log.u_bbr.delRate = le->m_head->m_flags;
178 log.u_bbr.rttProp = le->m_head->m_pkthdr.rcv_tstmp;
182 log.u_bbr.timeStamp = cts;
183 log.u_bbr.epoch = le->next_seq;
184 log.u_bbr.lt_epoch = le->ack_seq;
186 log.u_bbr.cwnd_gain = le->window;
189 log.u_bbr.bw_inuse = (uintptr_t)le->m_head;
190 bintime2timeval(&lc->lro_last_queue_time, &btv);
192 log.u_bbr.flex7 = le->compressed;
193 log.u_bbr.pacing_gain = le->uncompressed;
198 TCP_LOG_EVENTP(tp, NULL, &tptosocket(tp)->so_rcv,
199 &tptosocket(tp)->so_snd,
213 m = STAILQ_LAST(&tp->t_inqueue, mbuf, m_stailqpkt);
214 if (m != NULL && (m->m_flags & M_ACKCMP) != 0) {
224 tp->t_flags2 |= TF2_MBUF_L_ACKS;
232 if (tp->t_flags2 & TF2_MBUF_L_ACKS)
242 m->m_pkthdr.rcvif = lc->ifp;
243 m->m_flags |= M_ACKCMP;
280 ETHER_BPF_MTAP(lc->ifp, m);
285 tcp_hdr_offset = m->m_pkthdr.lro_tcp_h_off;
286 lro_type = le->inner.data.lro_type;
289 lro_type = le->outer.data.lro_type;
292 tcp_hdr_offset -= sizeof(*le->outer.ip4);
293 m->m_pkthdr.lro_etype = ETHERTYPE_IP;
294 IP_PROBE(receive, NULL, NULL, le->outer.ip4, lc->ifp,
295 le->outer.ip4, NULL);
298 tcp_hdr_offset -= sizeof(*le->outer.ip6);
299 m->m_pkthdr.lro_etype = ETHERTYPE_IPV6;
300 IP_PROBE(receive, NULL, NULL, le->outer.ip6, lc->ifp,
301 NULL, le->outer.ip6);
308 switch (le->outer.data.lro_type) {
310 IP_PROBE(receive, NULL, NULL, le->outer.ip4, lc->ifp,
311 le->outer.ip4, NULL);
312 UDP_PROBE(receive, NULL, NULL, le->outer.ip4, NULL,
313 le->outer.udp);
316 IP_PROBE(receive, NULL, NULL, le->outer.ip6, lc->ifp,
317 NULL, le->outer.ip6);
318 UDP_PROBE(receive, NULL, NULL, le->outer.ip6, NULL,
319 le->outer.udp);
325 tcp_hdr_offset -= sizeof(*le->outer.ip4);
326 m->m_pkthdr.lro_etype = ETHERTYPE_IP;
327 IP_PROBE(receive, NULL, NULL, le->inner.ip4, NULL,
328 le->inner.ip4, NULL);
331 switch (le->outer.data.lro_type) {
333 IP_PROBE(receive, NULL, NULL, le->outer.ip4, lc->ifp,
334 le->outer.ip4, NULL);
335 UDP_PROBE(receive, NULL, NULL, le->outer.ip4, NULL,
336 le->outer.udp);
339 IP_PROBE(receive, NULL, NULL, le->outer.ip6, lc->ifp,
340 NULL, le->outer.ip6);
341 UDP_PROBE(receive, NULL, NULL, le->outer.ip6, NULL,
342 le->outer.udp);
348 tcp_hdr_offset -= sizeof(*le->outer.ip6);
349 m->m_pkthdr.lro_etype = ETHERTYPE_IPV6;
350 IP_PROBE(receive, NULL, NULL, le->inner.ip6, NULL, NULL,
351 le->inner.ip6);
360 m->m_flags |= M_LRO_EHDRSTRP;
361 m->m_flags &= ~M_ACKCMP;
362 m->m_pkthdr.lro_tcp_h_off -= tcp_hdr_offset;
366 th->th_sum = 0; /* TCP checksum is valid. */
375 ((tp->t_flags2 & TF2_DONT_SACK_QUEUE) == 0)) {
383 /* Wakeup override check if we are false here */
390 if ((tp->t_flags2 & TF2_MBUF_ACKCMP) == 0)
397 iptos = l3.ip4->ip_tos;
416 * Link in the new cmp ack to our in-order place,
419 nm->m_nextpkt = m;
430 pp = &nm->m_nextpkt;
437 tp->t_flags2 |= TF2_MBUF_L_ACKS;
443 le->compressed++;
446 idx = (nm->m_len / sizeof(struct tcp_ackent));
449 /* Bump the size of both pkt-hdr and len */
450 nm->m_len += sizeof(struct tcp_ackent);
451 nm->m_pkthdr.len += sizeof(struct tcp_ackent);
454 *pp = m->m_nextpkt;
455 m->m_nextpkt = NULL;
460 le->uncompressed++;
470 STAILQ_HEAD(, mbuf) q = { le->m_head,
471 &STAILQ_NEXT(le->m_last_mbuf, m_stailqpkt) };
472 STAILQ_CONCAT(&tp->t_inqueue, &q);
473 le->m_head = NULL;
474 le->m_last_mbuf = NULL;
483 switch (pa->data.lro_type) {
487 &pa->data.s_addr.v6,
488 pa->data.s_port,
489 &pa->data.d_addr.v6,
490 pa->data.d_port,
498 pa->data.s_addr.v4,
499 pa->data.s_port,
500 pa->data.d_addr.v4,
501 pa->data.d_port,
523 (le->outer.data.vlan_id != 0) ||
524 (le->inner.data.lro_type != LRO_TYPE_NONE))
530 * we use all-zero to indicate unbounded/unconnected pcb,
536 if (__predict_false(le->outer.data.lro_type == LRO_TYPE_IPV6_TCP &&
537 IN6_IS_ADDR_UNSPECIFIED(&le->outer.data.s_addr.v6)))
540 if (__predict_false(le->inner.data.lro_type == LRO_TYPE_IPV6_TCP &&
541 IN6_IS_ADDR_UNSPECIFIED(&le->inner.data.s_addr.v6)))
545 CURVNET_SET(lc->ifp->if_vnet);
563 tp = tcp_lro_lookup(lc->ifp,
564 (le->inner.data.lro_type == LRO_TYPE_NONE) ? &le->outer : &le->inner);
572 if (tp->t_state == TCPS_TIME_WAIT) {
576 if (tp->t_lro_cpu == HPTS_CPU_NONE && lc->lro_cpu_is_set == 1)
577 tp->t_lro_cpu = lc->lro_last_cpu;
579 if ((tp->t_flags2 & (TF2_SUPPORTS_MBUFQ | TF2_MBUF_ACKCMP)) == 0) {
584 if (tp->t_flags2 & TF2_MBUF_QUEUE_READY)
589 bpf_req = bpf_peers_present(lc->ifp->if_bpf);
592 if (lc->ifp->if_type == IFT_IEEE8023ADLAG ||
593 lc->ifp->if_type == IFT_INFINIBANDLAG) {
594 struct lagg_port *lp = lc->ifp->if_lagg;
595 struct lagg_softc *sc = lp->lp_softc;
597 lagg_ifp = sc->sc_ifp;
599 lagg_bpf_req = bpf_peers_present(lagg_ifp->if_bpf);
605 for (pp = &le->m_head; *pp != NULL; ) {
611 pp = &(*pp)->m_nextpkt;
628 pp = &mv_to->m_nextpkt;
638 if (pp == &le->m_head)
639 le->m_last_mbuf = *pp;
641 le->m_last_mbuf = __containerof(pp, struct mbuf, m_nextpkt);
644 if (le->m_head != NULL) {
647 tcp_lro_log(tp, lc, le, NULL, 22, 1, tp->t_flags2, 0, 1);
654 if ((*tp->t_fb->tfb_do_queued_segments)(tp, 0))