1 /* $NetBSD: in_offload.c,v 1.15 2024/07/05 04:31:54 rin Exp $ */
2
3 /*
4 * Copyright (c)2005, 2006 YAMAMOTO Takashi,
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 __KERNEL_RCSID(0, "$NetBSD: in_offload.c,v 1.15 2024/07/05 04:31:54 rin Exp $");
31
32 #include <sys/param.h>
33 #include <sys/mbuf.h>
34
35 #include <net/if.h>
36
37 #include <netinet/in.h>
38 #include <netinet/in_systm.h>
39 #include <netinet/ip.h>
40 #include <netinet/ip_var.h>
41 #include <netinet/tcp.h>
42 #include <netinet/in_offload.h>
43
44 /*
45 * Handle M_CSUM_TSOv4 in software. Split the TCP payload in chunks of
46 * size MSS, and return mbuf chain consists of them.
47 */
48 struct mbuf *
tcp4_segment(struct mbuf * m,int off)49 tcp4_segment(struct mbuf *m, int off)
50 {
51 int mss;
52 int iphlen, thlen;
53 int hlen, len;
54 struct ip *ip;
55 struct tcphdr *th;
56 uint16_t ipid, phsum;
57 uint32_t tcpseq;
58 struct mbuf *hdr = NULL;
59 struct mbuf *m0 = NULL;
60 struct mbuf *prev = NULL;
61 struct mbuf *n, *t;
62 int nsegs;
63
64 KASSERT((m->m_flags & M_PKTHDR) != 0);
65 KASSERT((m->m_pkthdr.csum_flags & M_CSUM_TSOv4) != 0);
66
67 m->m_pkthdr.csum_flags = 0;
68
69 len = m->m_pkthdr.len;
70 KASSERT(len >= off + sizeof(*ip) + sizeof(*th));
71
72 hlen = off + sizeof(*ip);
73 if (m->m_len < hlen) {
74 m = m_pullup(m, hlen);
75 if (m == NULL)
76 goto quit;
77 }
78 ip = (void *)(mtod(m, char *) + off);
79 iphlen = ip->ip_hl * 4;
80 KASSERT(ip->ip_v == IPVERSION);
81 KASSERT(iphlen >= sizeof(*ip));
82 KASSERT(ip->ip_p == IPPROTO_TCP);
83 ipid = ntohs(ip->ip_id);
84
85 hlen = off + iphlen + sizeof(*th);
86 if (m->m_len < hlen) {
87 m = m_pullup(m, hlen);
88 if (m == NULL)
89 goto quit;
90 }
91 th = (void *)(mtod(m, char *) + off + iphlen);
92 tcpseq = ntohl(th->th_seq);
93 thlen = th->th_off * 4;
94 hlen = off + iphlen + thlen;
95
96 mss = m->m_pkthdr.segsz;
97 KASSERT(mss != 0);
98 KASSERT(len > hlen);
99
100 t = m_split(m, hlen, M_NOWAIT);
101 if (t == NULL)
102 goto quit;
103 hdr = m;
104 m = t;
105
106 len -= hlen;
107 KASSERT(len % mss == 0);
108
109 ip = (void *)(mtod(hdr, char *) + off);
110 ip->ip_len = htons(iphlen + thlen + mss);
111 phsum = in_cksum_phdr(ip->ip_src.s_addr, ip->ip_dst.s_addr,
112 htons((uint16_t)(thlen + mss) + IPPROTO_TCP));
113
114 for (nsegs = len / mss; nsegs > 0; nsegs--) {
115 if (nsegs > 1) {
116 n = m_dup(hdr, 0, hlen, M_NOWAIT);
117 if (n == NULL)
118 goto quit;
119 } else
120 n = hdr;
121 KASSERT(n->m_len == hlen); /* XXX */
122
123 if (nsegs > 1) {
124 t = m_split(m, mss, M_NOWAIT);
125 if (t == NULL) {
126 m_freem(n);
127 goto quit;
128 }
129 } else
130 t = m;
131 m_cat(n, m);
132 m = t;
133
134 KASSERT(n->m_len >= hlen); /* XXX */
135
136 if (m0 == NULL)
137 m0 = n;
138
139 if (prev != NULL)
140 prev->m_nextpkt = n;
141
142 n->m_pkthdr.len = hlen + mss;
143 n->m_nextpkt = NULL; /* XXX */
144
145 ip = (void *)(mtod(n, char *) + off);
146 ip->ip_id = htons(ipid);
147 ip->ip_sum = 0;
148 ip->ip_sum = in4_cksum(n, 0, off, iphlen);
149
150 th = (void *)(mtod(n, char *) + off + iphlen);
151 th->th_seq = htonl(tcpseq);
152 th->th_sum = phsum;
153 th->th_sum = in4_cksum(n, 0, off + iphlen, thlen + mss);
154
155 tcpseq += mss;
156 ipid++;
157 prev = n;
158 }
159 return m0;
160
161 quit:
162 m_freem(hdr);
163 m_freem(m);
164 for (m = m0; m != NULL; m = n) {
165 n = m->m_nextpkt;
166 m_freem(m);
167 }
168
169 return NULL;
170 }
171
172 int
ip_tso_output(struct ifnet * ifp,struct mbuf * m,const struct sockaddr * sa,struct rtentry * rt)173 ip_tso_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *sa,
174 struct rtentry *rt)
175 {
176 struct mbuf *n;
177 int error = 0;
178
179 m = tcp4_segment(m, 0);
180 if (m == NULL)
181 return ENOMEM;
182 do {
183 n = m->m_nextpkt;
184 if (error == 0)
185 error = ip_if_output(ifp, m, sa, rt);
186 else
187 m_freem(m);
188 m = n;
189 } while (m != NULL);
190 return error;
191 }
192
193 /*
194 * Compute now in software the IP and TCP/UDP checksums. Cancel the
195 * hardware offloading.
196 */
197 void
in_undefer_cksum(struct mbuf * mh,size_t hdrlen,int csum_flags)198 in_undefer_cksum(struct mbuf *mh, size_t hdrlen, int csum_flags)
199 {
200 const size_t iphdrlen = M_CSUM_DATA_IPv4_IPHL(mh->m_pkthdr.csum_data);
201 uint16_t csum;
202 uint16_t ip_len;
203 uint16_t *csump;
204 struct mbuf *m = mh;
205
206 KASSERT(mh->m_flags & M_PKTHDR);
207 KASSERT(mh->m_pkthdr.len > hdrlen);
208 KASSERT((mh->m_pkthdr.csum_flags & csum_flags) == csum_flags);
209
210 /*
211 * Deal with prepended frame header as done by e.g. ether_output().
212 * If first mbuf in chain has just the header, use second mbuf
213 * for the actual checksum. in4_csum() expects the passed mbuf
214 * to have the whole (struct ip) area contiguous.
215 */
216 if (m->m_len <= hdrlen) {
217 hdrlen -= m->m_len;
218 m = m->m_next;
219 KASSERT(m != NULL);
220 }
221
222 if (__predict_true(hdrlen + sizeof(struct ip) <= m->m_len)) {
223 struct ip *ip = (struct ip *)(mtod(m, uint8_t *) + hdrlen);
224
225 ip_len = ip->ip_len;
226 csump = &ip->ip_sum;
227 } else {
228 const size_t ip_len_offset =
229 hdrlen + offsetof(struct ip, ip_len);
230
231 m_copydata(m, ip_len_offset, sizeof(ip_len), &ip_len);
232 csump = NULL;
233 }
234 ip_len = ntohs(ip_len);
235
236 if (csum_flags & M_CSUM_IPv4) {
237 csum = in4_cksum(m, 0, hdrlen, iphdrlen);
238 if (csump != NULL) {
239 *csump = csum;
240 } else {
241 const size_t offset = hdrlen +
242 offsetof(struct ip, ip_sum);
243
244 m_copyback(m, offset, sizeof(uint16_t), &csum);
245 }
246 }
247
248 if (csum_flags & (M_CSUM_UDPv4|M_CSUM_TCPv4)) {
249 size_t l4offset = hdrlen + iphdrlen;
250
251 csum = in4_cksum(m, 0, l4offset, ip_len - iphdrlen);
252 if (csum == 0 && (csum_flags & M_CSUM_UDPv4) != 0)
253 csum = 0xffff;
254
255 l4offset += M_CSUM_DATA_IPv4_OFFSET(m->m_pkthdr.csum_data);
256
257 if (__predict_true(l4offset + sizeof(uint16_t) <= m->m_len)) {
258 *(uint16_t *)(mtod(m, char *) + l4offset) = csum;
259 } else {
260 m_copyback(m, l4offset, sizeof(csum), (void *)&csum);
261 }
262 }
263
264 mh->m_pkthdr.csum_flags ^= csum_flags;
265 }
266
267 /*
268 * Compute now in software the TCP/UDP checksum. Cancel the hardware
269 * offloading.
270 */
271 void
in_undefer_cksum_tcpudp(struct mbuf * m)272 in_undefer_cksum_tcpudp(struct mbuf *m)
273 {
274 struct ip *ip;
275 uint16_t csum, offset;
276
277 KASSERT((m->m_flags & M_PKTHDR) != 0);
278 KASSERT((m->m_pkthdr.csum_flags & (M_CSUM_TCPv4|M_CSUM_UDPv4)) != 0);
279 KASSERT((m->m_pkthdr.csum_flags & (M_CSUM_TCPv6|M_CSUM_UDPv6)) == 0);
280
281 ip = mtod(m, struct ip *);
282 offset = ip->ip_hl << 2;
283
284 csum = in4_cksum(m, 0, offset, ntohs(ip->ip_len) - offset);
285 if (csum == 0 && (m->m_pkthdr.csum_flags & M_CSUM_UDPv4) != 0)
286 csum = 0xffff;
287
288 offset += M_CSUM_DATA_IPv4_OFFSET(m->m_pkthdr.csum_data);
289
290 if ((offset + sizeof(uint16_t)) <= m->m_len) {
291 *(uint16_t *)(mtod(m, char *) + offset) = csum;
292 } else {
293 m_copyback(m, offset, sizeof(csum), (void *)&csum);
294 }
295 }
296