1 /* $NetBSD: copyout.c,v 1.10 2022/05/22 11:27:34 andvar Exp $ */
2
3 /*-
4 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects
9 * Agency and which was developed by Matt Thomas of 3am Software Foundry.
10 *
11 * This material is based upon work supported by the Defense Advanced Research
12 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under
13 * Contract No. N66001-09-C-2073.
14 * Approved for Public Release, Distribution Unlimited
15 *
16 * Redistribution and use in source and binary forms, with or without
17 * modification, are permitted provided that the following conditions
18 * are met:
19 * 1. Redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer.
21 * 2. Redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
26 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 * POSSIBILITY OF SUCH DAMAGE.
36 */
37
38 #define __UFETCHSTORE_PRIVATE
39
40 #include <sys/cdefs.h>
41 __KERNEL_RCSID(0, "$NetBSD: copyout.c,v 1.10 2022/05/22 11:27:34 andvar Exp $");
42
43 #include <sys/param.h>
44 #include <sys/lwp.h>
45 #include <sys/systm.h>
46
47 #include <powerpc/pcb.h>
48
49 #include <powerpc/booke/cpuvar.h>
50
51 static inline void
copyout_uint8(uint8_t * udaddr,uint8_t data,register_t ds_msr)52 copyout_uint8(uint8_t *udaddr, uint8_t data, register_t ds_msr)
53 {
54 register_t msr;
55 __asm volatile(
56 "mfmsr %[msr]" /* Save MSR */
57 "\n\t" "mtmsr %[ds_msr]; sync; isync" /* DS on */
58 "\n\t" "stb %[data],0(%[udaddr])" /* store user byte */
59 "\n\t" "mtmsr %[msr]; sync; isync" /* DS off */
60 : [msr] "=&r" (msr)
61 : [ds_msr] "r" (ds_msr), [data] "r" (data), [udaddr] "b" (udaddr));
62 }
63
64 static inline void
copyout_uint16(uint16_t * udaddr,uint8_t data,register_t ds_msr)65 copyout_uint16(uint16_t *udaddr, uint8_t data, register_t ds_msr)
66 {
67 register_t msr;
68 __asm volatile(
69 "mfmsr %[msr]" /* Save MSR */
70 "\n\t" "mtmsr %[ds_msr]; sync; isync" /* DS on */
71 "\n\t" "sth %[data],0(%[udaddr])" /* store user half */
72 "\n\t" "mtmsr %[msr]; sync; isync" /* DS off */
73 : [msr] "=&r" (msr)
74 : [ds_msr] "r" (ds_msr), [data] "r" (data), [udaddr] "b" (udaddr));
75 }
76
77 static inline void
copyout_uint32(uint32_t * const udaddr,uint32_t data,register_t ds_msr)78 copyout_uint32(uint32_t * const udaddr, uint32_t data, register_t ds_msr)
79 {
80 register_t msr;
81 __asm volatile(
82 "mfmsr %[msr]" /* Save MSR */
83 "\n\t" "mtmsr %[ds_msr]; sync; isync" /* DS on */
84 "\n\t" "stw %[data],0(%[udaddr])" /* store user data */
85 "\n\t" "mtmsr %[msr]; sync; isync" /* DS off */
86 : [msr] "=&r" (msr)
87 : [ds_msr] "r" (ds_msr), [data] "r" (data), [udaddr] "b" (udaddr));
88 }
89
90 #if 0
91 static inline void
92 copyout_le32(uint32_t * const udaddr, uint32_t data, register_t ds_msr)
93 {
94 register_t msr;
95 __asm volatile(
96 "mfmsr %[msr]" /* Save MSR */
97 "\n\t" "mtmsr %[ds_msr]; sync; isync" /* DS on */
98 "\n\t" "stwbrx %[data],0,%[udaddr]" /* store user data */
99 "\n\t" "mtmsr %[msr]; sync; isync" /* DS off */
100 : [msr] "=&r" (msr)
101 : [ds_msr] "r" (ds_msr), [data] "r" (data), [udaddr] "b" (udaddr));
102 }
103
104 static inline void
105 copyout_le32_with_mask(uint32_t * const udaddr, uint32_t data,
106 uint32_t mask, register_t ds_msr)
107 {
108 register_t msr;
109 uint32_t tmp;
110 KASSERT((data & ~mask) == 0);
111 __asm volatile(
112 "mfmsr %[msr]" /* Save MSR */
113 "\n\t" "mtmsr %[ds_msr]; sync; isync" /* DS on */
114 "\n\t" "lwbrx %[tmp],0,%[udaddr]" /* fetch user data */
115 "\n\t" "andc %[tmp],%[tmp],%[mask]" /* mask out new data */
116 "\n\t" "or %[tmp],%[tmp],%[data]" /* merge new data */
117 "\n\t" "stwbrx %[tmp],0,%[udaddr]" /* store user data */
118 "\n\t" "mtmsr %[msr]; sync; isync" /* DS off */
119 : [msr] "=&r" (msr), [tmp] "=&r" (tmp)
120 : [ds_msr] "r" (ds_msr), [data] "r" (data),
121 [mask] "r" (mask), [udaddr] "b" (udaddr));
122 }
123 #endif
124
125 static inline void
copyout_16uint8s(const uint8_t * ksaddr8,uint8_t * udaddr8,register_t ds_msr)126 copyout_16uint8s(const uint8_t *ksaddr8, uint8_t *udaddr8, register_t ds_msr)
127 {
128 register_t msr;
129 __asm volatile(
130 "mfmsr %[msr]" /* Save MSR */
131 "\n\t" "mtmsr %[ds_msr]; sync; isync" /* DS on */
132 "\n\t" "stb %[data0],0(%[udaddr8])" /* store user data */
133 "\n\t" "stb %[data1],1(%[udaddr8])" /* store user data */
134 "\n\t" "stb %[data2],2(%[udaddr8])" /* store user data */
135 "\n\t" "stb %[data3],3(%[udaddr8])" /* store user data */
136 "\n\t" "stb %[data4],4(%[udaddr8])" /* store user data */
137 "\n\t" "stb %[data5],5(%[udaddr8])" /* store user data */
138 "\n\t" "stb %[data6],6(%[udaddr8])" /* store user data */
139 "\n\t" "stb %[data7],7(%[udaddr8])" /* store user data */
140 "\n\t" "stb %[data8],8(%[udaddr8])" /* store user data */
141 "\n\t" "stb %[data9],9(%[udaddr8])" /* store user data */
142 "\n\t" "stb %[data10],10(%[udaddr8])" /* store user data */
143 "\n\t" "stb %[data11],11(%[udaddr8])" /* store user data */
144 "\n\t" "stb %[data12],12(%[udaddr8])" /* store user data */
145 "\n\t" "stb %[data13],13(%[udaddr8])" /* store user data */
146 "\n\t" "stb %[data14],14(%[udaddr8])" /* store user data */
147 "\n\t" "stb %[data15],15(%[udaddr8])" /* store user data */
148 "\n\t" "mtmsr %[msr]; sync; isync" /* DS off */
149 : [msr] "=&r" (msr)
150 : [ds_msr] "r" (ds_msr), [udaddr8] "b" (udaddr8),
151 [data0] "r" (ksaddr8[0]), [data1] "r" (ksaddr8[1]),
152 [data2] "r" (ksaddr8[2]), [data3] "r" (ksaddr8[3]),
153 [data4] "r" (ksaddr8[4]), [data5] "r" (ksaddr8[5]),
154 [data6] "r" (ksaddr8[6]), [data7] "r" (ksaddr8[7]),
155 [data8] "r" (ksaddr8[8]), [data9] "r" (ksaddr8[9]),
156 [data10] "r" (ksaddr8[10]), [data11] "r" (ksaddr8[11]),
157 [data12] "r" (ksaddr8[12]), [data13] "r" (ksaddr8[13]),
158 [data14] "r" (ksaddr8[14]), [data15] "r" (ksaddr8[15]));
159 }
160
161 static inline void
copyout_8uint32s(const uint32_t * const ksaddr32,uint32_t * const udaddr32,const register_t ds_msr,const size_t line_mask)162 copyout_8uint32s(const uint32_t * const ksaddr32, uint32_t * const udaddr32,
163 const register_t ds_msr, const size_t line_mask)
164 {
165 register_t msr;
166 register_t tmp;
167 __asm volatile(
168 "and. %[tmp],%[line_mask],%[udaddr32]"
169 "\n\t" "mfmsr %[msr]" /* Save MSR */
170 "\n\t" "mtmsr %[ds_msr]; sync; isync" /* DS on */
171 "\n\t" "bne 0,1f"
172 "\n\t" "dcba 0,%[udaddr32]"
173 "\n" "1:"
174 "\n\t" "stw %[data0],0(%[udaddr32])" /* store user data */
175 "\n\t" "stw %[data1],4(%[udaddr32])" /* store user data */
176 "\n\t" "stw %[data2],8(%[udaddr32])" /* store user data */
177 "\n\t" "stw %[data3],12(%[udaddr32])" /* store user data */
178 "\n\t" "stw %[data4],16(%[udaddr32])" /* store user data */
179 "\n\t" "stw %[data5],20(%[udaddr32])" /* store user data */
180 "\n\t" "stw %[data6],24(%[udaddr32])" /* store user data */
181 "\n\t" "stw %[data7],28(%[udaddr32])" /* store user data */
182 "\n\t" "mtmsr %[msr]; sync; isync" /* DS off */
183 : [msr] "=&r" (msr), [tmp] "=&r" (tmp)
184 : [ds_msr] "r" (ds_msr), [udaddr32] "b" (udaddr32),
185 [line_mask] "r" (line_mask),
186 [data0] "r" (ksaddr32[0]), [data1] "r" (ksaddr32[1]),
187 [data2] "r" (ksaddr32[2]), [data3] "r" (ksaddr32[3]),
188 [data4] "r" (ksaddr32[4]), [data5] "r" (ksaddr32[5]),
189 [data6] "r" (ksaddr32[6]), [data7] "r" (ksaddr32[7])
190 : "cr0");
191 }
192
193 static inline void
copyout_16uint32s(const uint32_t * const ksaddr32,uint32_t * const udaddr32,const register_t ds_msr,const size_t line_mask)194 copyout_16uint32s(const uint32_t * const ksaddr32, uint32_t * const udaddr32,
195 const register_t ds_msr, const size_t line_mask)
196 {
197 KASSERT(((uintptr_t)udaddr32 & line_mask) == 0);
198 register_t msr;
199 register_t tmp;
200 __asm volatile(
201 "and. %[tmp],%[line_mask],%[udaddr32]"
202 "\n\t" "cmplwi 2,%[line_size],32"
203 "\n\t" "mfmsr %[msr]" /* Save MSR */
204 "\n\t" "mtmsr %[ds_msr]; sync; isync" /* DS on */
205 "\n\t" "bne 0,1f"
206 "\n\t" "dcba 0,%[udaddr32]"
207 "\n\t" "bne 2,1f"
208 "\n\t" "dcba %[line_size],%[udaddr32]"
209 "\n" "1:"
210 "\n\t" "stw %[data0],0(%[udaddr32])" /* store user data */
211 "\n\t" "stw %[data1],4(%[udaddr32])" /* store user data */
212 "\n\t" "stw %[data2],8(%[udaddr32])" /* store user data */
213 "\n\t" "stw %[data3],12(%[udaddr32])" /* store user data */
214 "\n\t" "stw %[data4],16(%[udaddr32])" /* store user data */
215 "\n\t" "stw %[data5],20(%[udaddr32])" /* store user data */
216 "\n\t" "stw %[data6],24(%[udaddr32])" /* store user data */
217 "\n\t" "stw %[data7],28(%[udaddr32])" /* store user data */
218 "\n\t" "stw %[data8],32(%[udaddr32])" /* store user data */
219 "\n\t" "stw %[data9],36(%[udaddr32])" /* store user data */
220 "\n\t" "stw %[data10],40(%[udaddr32])" /* store user data */
221 "\n\t" "stw %[data11],44(%[udaddr32])" /* store user data */
222 "\n\t" "stw %[data12],48(%[udaddr32])" /* store user data */
223 "\n\t" "stw %[data13],52(%[udaddr32])" /* store user data */
224 "\n\t" "stw %[data14],56(%[udaddr32])" /* store user data */
225 "\n\t" "stw %[data15],60(%[udaddr32])" /* store user data */
226 "\n\t" "mtmsr %[msr]; sync; isync" /* DS off */
227 : [msr] "=&r" (msr), [tmp] "=&r" (tmp)
228 : [ds_msr] "r" (ds_msr), [udaddr32] "b" (udaddr32),
229 [line_size] "r" (line_mask + 1), [line_mask] "r" (line_mask),
230 [data0] "r" (ksaddr32[0]), [data1] "r" (ksaddr32[1]),
231 [data2] "r" (ksaddr32[2]), [data3] "r" (ksaddr32[3]),
232 [data4] "r" (ksaddr32[4]), [data5] "r" (ksaddr32[5]),
233 [data6] "r" (ksaddr32[6]), [data7] "r" (ksaddr32[7]),
234 [data8] "r" (ksaddr32[8]), [data9] "r" (ksaddr32[9]),
235 [data10] "r" (ksaddr32[10]), [data11] "r" (ksaddr32[11]),
236 [data12] "r" (ksaddr32[12]), [data13] "r" (ksaddr32[13]),
237 [data14] "r" (ksaddr32[14]), [data15] "r" (ksaddr32[15])
238 : "cr0", "cr2");
239 }
240
241 static inline void
copyout_uint8s(vaddr_t ksaddr,vaddr_t udaddr,size_t len,register_t ds_msr)242 copyout_uint8s(vaddr_t ksaddr, vaddr_t udaddr, size_t len, register_t ds_msr)
243 {
244 const uint8_t *ksaddr8 = (void *)ksaddr;
245 uint8_t *udaddr8 = (void *)udaddr;
246
247 __builtin_prefetch(ksaddr8, 0, 1);
248
249 for (; len >= 16; len -= 16, ksaddr8 += 16, udaddr8 += 16) {
250 __builtin_prefetch(ksaddr8 + 16, 0, 1);
251 copyout_16uint8s(ksaddr8, udaddr8, ds_msr);
252 }
253
254 while (len-- > 0) {
255 copyout_uint8(udaddr8++, *ksaddr8++, ds_msr);
256 }
257 }
258
259 static inline void
copyout_uint32s(vaddr_t ksaddr,vaddr_t udaddr,size_t len,register_t ds_msr)260 copyout_uint32s(vaddr_t ksaddr, vaddr_t udaddr, size_t len, register_t ds_msr)
261 {
262 const size_t line_size = curcpu()->ci_ci.dcache_line_size;
263 const size_t line_mask = line_size - 1;
264 const size_t udalignment = udaddr & line_mask;
265 KASSERT((ksaddr & 3) == 0);
266 KASSERT((udaddr & 3) == 0);
267 const uint32_t *ksaddr32 = (void *)ksaddr;
268 uint32_t *udaddr32 = (void *)udaddr;
269 len >>= 2;
270 __builtin_prefetch(ksaddr32, 0, 1);
271 if (udalignment != 0 && udalignment + 4*len > line_size) {
272 size_t slen = (line_size - udalignment) >> 2;
273 len -= slen;
274 for (; slen >= 8; ksaddr32 += 8, udaddr32 += 8, slen -= 8) {
275 copyout_8uint32s(ksaddr32, udaddr32, ds_msr, line_mask);
276 }
277 while (slen-- > 0) {
278 copyout_uint32(udaddr32++, *ksaddr32++, ds_msr);
279 }
280 if (len == 0)
281 return;
282 }
283 __builtin_prefetch(ksaddr32, 0, 1);
284 while (len >= 16) {
285 __builtin_prefetch(ksaddr32 + 8, 0, 1);
286 __builtin_prefetch(ksaddr32 + 16, 0, 1);
287 copyout_16uint32s(ksaddr32, udaddr32, ds_msr, line_mask);
288 ksaddr32 += 16, udaddr32 += 16, len -= 16;
289 }
290 KASSERT(len <= 16);
291 if (len >= 8) {
292 __builtin_prefetch(ksaddr32 + 8, 0, 1);
293 copyout_8uint32s(ksaddr32, udaddr32, ds_msr, line_mask);
294 ksaddr32 += 8, udaddr32 += 8, len -= 8;
295 }
296 while (len-- > 0) {
297 copyout_uint32(udaddr32++, *ksaddr32++, ds_msr);
298 }
299 }
300
301 int
_ustore_8(uint8_t * vusaddr,uint8_t val)302 _ustore_8(uint8_t *vusaddr, uint8_t val)
303 {
304 struct pcb * const pcb = lwp_getpcb(curlwp);
305 struct faultbuf env;
306
307 if (setfault(&env) != 0) {
308 pcb->pcb_onfault = NULL;
309 return EFAULT;
310 }
311
312 copyout_uint8(vusaddr, val, mfmsr() | PSL_DS);
313
314 pcb->pcb_onfault = NULL;
315
316 return 0;
317 }
318
319 int
_ustore_16(uint16_t * vusaddr,uint16_t val)320 _ustore_16(uint16_t *vusaddr, uint16_t val)
321 {
322 struct pcb * const pcb = lwp_getpcb(curlwp);
323 struct faultbuf env;
324
325 if (setfault(&env) != 0) {
326 pcb->pcb_onfault = NULL;
327 return EFAULT;
328 }
329
330 copyout_uint16(vusaddr, val, mfmsr() | PSL_DS);
331
332 pcb->pcb_onfault = NULL;
333
334 return 0;
335 }
336
337 int
_ustore_32(uint32_t * vusaddr,uint32_t val)338 _ustore_32(uint32_t *vusaddr, uint32_t val)
339 {
340 struct pcb * const pcb = lwp_getpcb(curlwp);
341 struct faultbuf env;
342
343 if (setfault(&env) != 0) {
344 pcb->pcb_onfault = NULL;
345 return EFAULT;
346 }
347
348 copyout_uint32(vusaddr, val, mfmsr() | PSL_DS);
349
350 pcb->pcb_onfault = NULL;
351
352 return 0;
353 }
354
355 int
copyout(const void * vksaddr,void * vudaddr,size_t len)356 copyout(const void *vksaddr, void *vudaddr, size_t len)
357 {
358 struct pcb * const pcb = lwp_getpcb(curlwp);
359 struct faultbuf env;
360 vaddr_t udaddr = (vaddr_t) vudaddr;
361 vaddr_t ksaddr = (vaddr_t) vksaddr;
362
363 if (__predict_false(len == 0)) {
364 return 0;
365 }
366
367 const register_t ds_msr = mfmsr() | PSL_DS;
368
369 int rv = setfault(&env);
370 if (rv != 0) {
371 pcb->pcb_onfault = NULL;
372 return rv;
373 }
374
375 if (__predict_false(len < 4)) {
376 copyout_uint8s(ksaddr, udaddr, len, ds_msr);
377 pcb->pcb_onfault = NULL;
378 return 0;
379 }
380
381 const size_t alignment = (udaddr ^ ksaddr) & 3;
382 if (__predict_true(alignment == 0)) {
383 size_t slen;
384 if (__predict_false(ksaddr & 3)) {
385 slen = 4 - (ksaddr & 3);
386 copyout_uint8s(ksaddr, udaddr, slen, ds_msr);
387 udaddr += slen, ksaddr += slen, len -= slen;
388 }
389 slen = len & ~3;
390 if (__predict_true(slen >= 4)) {
391 copyout_uint32s(ksaddr, udaddr, slen, ds_msr);
392 udaddr += slen, ksaddr += slen, len -= slen;
393 }
394 }
395
396 if (len > 0) {
397 copyout_uint8s(ksaddr, udaddr, len, ds_msr);
398 }
399 pcb->pcb_onfault = NULL;
400 return 0;
401 }
402
403 #if 1
404 int
copyoutstr(const void * ksaddr,void * udaddr,size_t len,size_t * done)405 copyoutstr(const void *ksaddr, void *udaddr, size_t len, size_t *done)
406 {
407 struct pcb * const pcb = lwp_getpcb(curlwp);
408 struct faultbuf env;
409 int rv;
410
411 if (__predict_false(len == 0)) {
412 if (done)
413 *done = 0;
414 return 0;
415 }
416
417 rv = setfault(&env);
418 if (rv != 0) {
419 pcb->pcb_onfault = NULL;
420 if (done)
421 *done = 0;
422 return rv;
423 }
424
425 const register_t ds_msr = mfmsr() | PSL_DS;
426 const uint8_t *ksaddr8 = ksaddr;
427 size_t copylen = 0;
428
429 uint8_t *udaddr8 = (void *)udaddr;
430
431 while (copylen++ < len) {
432 const uint8_t data = *ksaddr8++;
433 copyout_uint8(udaddr8++, data, ds_msr);
434 if (data == 0)
435 goto out;
436 }
437 rv = ENAMETOOLONG;
438
439 out:
440 pcb->pcb_onfault = NULL;
441 if (done)
442 *done = copylen;
443 return rv;
444 }
445 #else
446 /* XXX This version of copyoutstr(9) has never been enabled so far. */
447 int
copyoutstr(const void * ksaddr,void * udaddr,size_t len,size_t * lenp)448 copyoutstr(const void *ksaddr, void *udaddr, size_t len, size_t *lenp)
449 {
450 struct pcb * const pcb = lwp_getpcb(curlwp);
451 struct faultbuf env;
452
453 if (__predict_false(len == 0)) {
454 if (lenp)
455 *lenp = 0;
456 return 0;
457 }
458
459 if (setfault(&env)) {
460 pcb->pcb_onfault = NULL;
461 if (lenp)
462 *lenp = 0;
463 return EFAULT;
464 }
465
466 const register_t ds_msr = mfmsr() | PSL_DS;
467 const uint8_t *ksaddr8 = ksaddr;
468 size_t copylen = 0;
469
470 uint32_t *udaddr32 = (void *)((uintptr_t)udaddr & ~3);
471
472 size_t boff = (uintptr_t)udaddr & 3;
473 bool done = false;
474 size_t wlen = 0;
475 size_t data = 0;
476
477 /*
478 * If the destination buffer doesn't start on a 32-bit boundary
479 * try to partially fill in the first word. If we succeed we can
480 * finish writing it while preserving the bytes on front.
481 */
482 if (boff > 0) {
483 KASSERT(len > 0);
484 do {
485 data = (data << 8) | *ksaddr8++;
486 wlen++;
487 done = ((uint8_t)data == 0 || len == wlen);
488 } while (!done && boff + wlen < 4);
489 KASSERT(wlen > 0);
490 data <<= 8 * boff;
491 if (!done || boff + wlen == 4) {
492 uint32_t mask = 0xffffffff << (8 * boff);
493 copyout_le32_with_mask(udaddr32++, data, mask, ds_msr);
494 boff = 0;
495 copylen = wlen;
496 wlen = 0;
497 data = 0;
498 }
499 }
500
501 /*
502 * Now we get to the heart of the routine. Build up complete words
503 * if possible. When we have one, write it to the user's address
504 * space and go for the next. If we ran out of space or we found the
505 * end of the string, stop building. If we managed to build a complete
506 * word, just write it and be happy. Otherwise we have to deal with
507 * the trailing bytes.
508 */
509 KASSERT(done || boff == 0);
510 KASSERT(done || copylen < len);
511 while (!done) {
512 KASSERT(wlen == 0);
513 KASSERT(copylen < len);
514 do {
515 data = (data << 8) | *ksaddr8++;
516 wlen++;
517 done = ((uint8_t)data == 0 || copylen + wlen == len);
518 } while (!done && wlen < 4);
519 KASSERT(done || wlen == 4);
520 if (__predict_true(wlen == 4)) {
521 copyout_le32(udaddr32++, data, ds_msr);
522 data = 0;
523 copylen += wlen;
524 wlen = 0;
525 KASSERT(copylen < len || done);
526 }
527 }
528 KASSERT(wlen < 3);
529 if (wlen) {
530 /*
531 * Remember even though we are running big-endian we are using
532 * byte reversed load/stores so we need to deal with things as
533 * little endian.
534 *
535 * wlen=1 boff=0:
536 * (~(~0 << 8) << 0) -> (~(0xffffff00) << 0) -> 0x000000ff
537 * wlen=1 boff=1:
538 * (~(~0 << 8) << 8) -> (~(0xffffff00) << 8) -> 0x0000ff00
539 * wlen=1 boff=2:
540 * (~(~0 << 8) << 16) -> (~(0xffffff00) << 16) -> 0x00ff0000
541 * wlen=1 boff=3:
542 * (~(~0 << 8) << 24) -> (~(0xffffff00) << 24) -> 0xff000000
543 * wlen=2 boff=0:
544 * (~(~0 << 16) << 0) -> (~(0xffff0000) << 0) -> 0x0000ffff
545 * wlen=2 boff=1:
546 * (~(~0 << 16) << 8) -> (~(0xffff0000) << 8) -> 0x00ffff00
547 * wlen=2 boff=2:
548 * (~(~0 << 16) << 16) -> (~(0xffff0000) << 16) -> 0xffff0000
549 * wlen=3 boff=0:
550 * (~(~0 << 24) << 0) -> (~(0xff000000) << 0) -> 0x00ffffff
551 * wlen=3 boff=1:
552 * (~(~0 << 24) << 8) -> (~(0xff000000) << 8) -> 0xffffff00
553 */
554 KASSERT(boff + wlen <= 4);
555 uint32_t mask = (~(~0 << (8 * wlen))) << (8 * boff);
556 KASSERT(mask != 0xffffffff);
557 copyout_le32_with_mask(udaddr32, data, mask, ds_msr);
558 copylen += wlen;
559 }
560
561 pcb->pcb_onfault = NULL;
562 if (lenp)
563 *lenp = copylen;
564 return 0;
565 }
566 #endif
567