1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause AND BSD-4-Clause
3 *
4 * Copyright (C) 2002 Benno Rice
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27 /*-
28 * Copyright (C) 1993 Wolfgang Solfrank.
29 * Copyright (C) 1993 TooLs GmbH.
30 * All rights reserved.
31 *
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
34 * are met:
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
40 * 3. All advertising materials mentioning features or use of this software
41 * must display the following acknowledgement:
42 * This product includes software developed by TooLs GmbH.
43 * 4. The name of TooLs GmbH may not be used to endorse or promote products
44 * derived from this software without specific prior written permission.
45 *
46 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
47 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
48 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
49 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
50 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
51 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
52 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
53 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
54 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
55 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
56 */
57
58 #include <sys/param.h>
59 #include <sys/lock.h>
60 #include <sys/mutex.h>
61 #include <sys/systm.h>
62 #include <sys/proc.h>
63
64 #include <vm/vm.h>
65 #include <vm/pmap.h>
66 #include <vm/vm_extern.h>
67 #include <vm/vm_map.h>
68
69 #include <machine/mmuvar.h>
70 #include <machine/pcb.h>
71 #include <machine/vmparam.h>
72 #include <machine/ifunc.h>
73
74 /*
75 * On powerpc64 (AIM only) the copy functions are IFUNCs, selecting the best
76 * option based on the PMAP in use.
77 *
78 * There are two options for copy functions on powerpc64:
79 * - 'remap' copies, which remap userspace segments into kernel space for
80 * copying. This is used by the 'oea64' pmap.
81 * - 'direct' copies, which copy directly from userspace. This does not require
82 * remapping user segments into kernel. This is used by the 'radix' pmap for
83 * performance.
84 *
85 * Book-E does not use the C 'remap' functions, opting instead to use the
86 * 'direct' copies, directly, avoiding the IFUNC overhead.
87 *
88 * On 32-bit AIM these functions bypass the IFUNC machinery for performance.
89 */
90 #ifdef __powerpc64__
91 int subyte_remap(volatile void *addr, int byte);
92 int subyte_direct(volatile void *addr, int byte);
93 int copyinstr_remap(const void *udaddr, void *kaddr, size_t len, size_t *done);
94 int copyinstr_direct(const void *udaddr, void *kaddr, size_t len, size_t *done);
95 int copyout_remap(const void *kaddr, void *udaddr, size_t len);
96 int copyout_direct(const void *kaddr, void *udaddr, size_t len);
97 int copyin_remap(const void *uaddr, void *kaddr, size_t len);
98 int copyin_direct(const void *uaddr, void *kaddr, size_t len);
99 int suword16_remap(volatile void *addr, int word);
100 int suword16_direct(volatile void *addr, int word);
101 int suword32_remap(volatile void *addr, int word);
102 int suword32_direct(volatile void *addr, int word);
103 int suword_remap(volatile void *addr, long word);
104 int suword_direct(volatile void *addr, long word);
105 int suword64_remap(volatile void *addr, int64_t word);
106 int suword64_direct(volatile void *addr, int64_t word);
107 int fubyte_remap(volatile const void *addr);
108 int fubyte_direct(volatile const void *addr);
109 int fuword16_remap(volatile const void *addr);
110 int fuword16_direct(volatile const void *addr);
111 int fueword32_remap(volatile const void *addr, int32_t *val);
112 int fueword32_direct(volatile const void *addr, int32_t *val);
113 int fueword64_remap(volatile const void *addr, int64_t *val);
114 int fueword64_direct(volatile const void *addr, int64_t *val);
115 int fueword_remap(volatile const void *addr, long *val);
116 int fueword_direct(volatile const void *addr, long *val);
117 int casueword32_remap(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp,
118 uint32_t new);
119 int casueword32_direct(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp,
120 uint32_t new);
121 int casueword_remap(volatile u_long *addr, u_long old, u_long *oldvalp,
122 u_long new);
123 int casueword_direct(volatile u_long *addr, u_long old, u_long *oldvalp,
124 u_long new);
125
126 /*
127 * The IFUNC resolver determines the copy based on whether the PMAP
128 * implementation includes a pmap_map_user_ptr function.
129 */
130 #define DEFINE_COPY_FUNC(ret, func, args) \
131 DEFINE_IFUNC(, ret, func, args) \
132 { \
133 return (PMAP_RESOLVE_FUNC(map_user_ptr) ? \
134 func##_remap : func##_direct); \
135 }
136 DEFINE_COPY_FUNC(int, subyte, (volatile void *, int))
137 DEFINE_COPY_FUNC(int, copyinstr, (const void *, void *, size_t, size_t *))
138 DEFINE_COPY_FUNC(int, copyin, (const void *, void *, size_t))
139 DEFINE_COPY_FUNC(int, copyout, (const void *, void *, size_t))
140 DEFINE_COPY_FUNC(int, suword, (volatile void *, long))
141 DEFINE_COPY_FUNC(int, suword16, (volatile void *, int))
142 DEFINE_COPY_FUNC(int, suword32, (volatile void *, int))
143 DEFINE_COPY_FUNC(int, suword64, (volatile void *, int64_t))
144 DEFINE_COPY_FUNC(int, fubyte, (volatile const void *))
145 DEFINE_COPY_FUNC(int, fuword16, (volatile const void *))
146 DEFINE_COPY_FUNC(int, fueword32, (volatile const void *, int32_t *))
147 DEFINE_COPY_FUNC(int, fueword64, (volatile const void *, int64_t *))
148 DEFINE_COPY_FUNC(int, fueword, (volatile const void *, long *))
149 DEFINE_COPY_FUNC(int, casueword32,
150 (volatile uint32_t *, uint32_t, uint32_t *, uint32_t))
151 DEFINE_COPY_FUNC(int, casueword, (volatile u_long *, u_long, u_long *, u_long))
152
153 #define REMAP(x) x##_remap
154 #else
155 #define REMAP(x) x
156 #endif
157
158 int
REMAP(copyout)159 REMAP(copyout)(const void *kaddr, void *udaddr, size_t len)
160 {
161 struct thread *td;
162 pmap_t pm;
163 jmp_buf env;
164 const char *kp;
165 char *up, *p;
166 size_t l;
167
168 td = curthread;
169 pm = &td->td_proc->p_vmspace->vm_pmap;
170
171 td->td_pcb->pcb_onfault = &env;
172 if (setjmp(env)) {
173 td->td_pcb->pcb_onfault = NULL;
174 return (EFAULT);
175 }
176
177 kp = kaddr;
178 up = udaddr;
179
180 while (len > 0) {
181 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) {
182 td->td_pcb->pcb_onfault = NULL;
183 return (EFAULT);
184 }
185
186 bcopy(kp, p, l);
187
188 up += l;
189 kp += l;
190 len -= l;
191 }
192
193 td->td_pcb->pcb_onfault = NULL;
194 return (0);
195 }
196
197 int
REMAP(copyin)198 REMAP(copyin)(const void *udaddr, void *kaddr, size_t len)
199 {
200 struct thread *td;
201 pmap_t pm;
202 jmp_buf env;
203 const char *up;
204 char *kp, *p;
205 size_t l;
206
207 td = curthread;
208 pm = &td->td_proc->p_vmspace->vm_pmap;
209
210 td->td_pcb->pcb_onfault = &env;
211 if (setjmp(env)) {
212 td->td_pcb->pcb_onfault = NULL;
213 return (EFAULT);
214 }
215
216 kp = kaddr;
217 up = udaddr;
218
219 while (len > 0) {
220 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) {
221 td->td_pcb->pcb_onfault = NULL;
222 return (EFAULT);
223 }
224
225 bcopy(p, kp, l);
226
227 up += l;
228 kp += l;
229 len -= l;
230 }
231
232 td->td_pcb->pcb_onfault = NULL;
233 return (0);
234 }
235
236 int
REMAP(copyinstr)237 REMAP(copyinstr)(const void *udaddr, void *kaddr, size_t len, size_t *done)
238 {
239 struct thread *td;
240 pmap_t pm;
241 jmp_buf env;
242 const char *up;
243 char *kp, *p;
244 size_t i, l, t;
245 int rv;
246
247 td = curthread;
248 pm = &td->td_proc->p_vmspace->vm_pmap;
249
250 t = 0;
251 rv = ENAMETOOLONG;
252
253 td->td_pcb->pcb_onfault = &env;
254 if (setjmp(env)) {
255 rv = EFAULT;
256 goto done;
257 }
258
259 kp = kaddr;
260 up = udaddr;
261
262 while (len > 0) {
263 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) {
264 rv = EFAULT;
265 goto done;
266 }
267
268 for (i = 0; len > 0 && i < l; i++, t++, len--) {
269 if ((*kp++ = *p++) == 0) {
270 i++, t++;
271 rv = 0;
272 goto done;
273 }
274 }
275
276 up += l;
277 }
278
279 done:
280 td->td_pcb->pcb_onfault = NULL;
281
282 if (done != NULL) {
283 *done = t;
284 }
285
286 return (rv);
287 }
288
289 int
REMAP(subyte)290 REMAP(subyte)(volatile void *addr, int byte)
291 {
292 struct thread *td;
293 pmap_t pm;
294 jmp_buf env;
295 char *p;
296
297 td = curthread;
298 pm = &td->td_proc->p_vmspace->vm_pmap;
299
300 td->td_pcb->pcb_onfault = &env;
301 if (setjmp(env)) {
302 td->td_pcb->pcb_onfault = NULL;
303 return (-1);
304 }
305
306 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
307 td->td_pcb->pcb_onfault = NULL;
308 return (-1);
309 }
310
311 *p = (char)byte;
312
313 td->td_pcb->pcb_onfault = NULL;
314 return (0);
315 }
316
317 int
REMAP(suword16)318 REMAP(suword16)(volatile void *addr, int word)
319 {
320 struct thread *td;
321 pmap_t pm;
322 jmp_buf env;
323 int16_t *p;
324
325 td = curthread;
326 pm = &td->td_proc->p_vmspace->vm_pmap;
327
328 td->td_pcb->pcb_onfault = &env;
329 if (setjmp(env)) {
330 td->td_pcb->pcb_onfault = NULL;
331 return (-1);
332 }
333
334 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
335 td->td_pcb->pcb_onfault = NULL;
336 return (-1);
337 }
338
339 *p = (int16_t)word;
340
341 td->td_pcb->pcb_onfault = NULL;
342 return (0);
343 }
344
345 #ifdef __powerpc64__
346 int
REMAP(suword32)347 REMAP(suword32)(volatile void *addr, int word)
348 {
349 struct thread *td;
350 pmap_t pm;
351 jmp_buf env;
352 int *p;
353
354 td = curthread;
355 pm = &td->td_proc->p_vmspace->vm_pmap;
356
357 td->td_pcb->pcb_onfault = &env;
358 if (setjmp(env)) {
359 td->td_pcb->pcb_onfault = NULL;
360 return (-1);
361 }
362
363 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
364 td->td_pcb->pcb_onfault = NULL;
365 return (-1);
366 }
367
368 *p = word;
369
370 td->td_pcb->pcb_onfault = NULL;
371 return (0);
372 }
373 #else
374 int
REMAP(suword32)375 REMAP(suword32)(volatile void *addr, int32_t word)
376 {
377 REMAP( return (suword)(addr, (long)word));
378 }
379 #endif
380
381 int
REMAP(suword)382 REMAP(suword)(volatile void *addr, long word)
383 {
384 struct thread *td;
385 pmap_t pm;
386 jmp_buf env;
387 long *p;
388
389 td = curthread;
390 pm = &td->td_proc->p_vmspace->vm_pmap;
391
392 td->td_pcb->pcb_onfault = &env;
393 if (setjmp(env)) {
394 td->td_pcb->pcb_onfault = NULL;
395 return (-1);
396 }
397
398 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
399 td->td_pcb->pcb_onfault = NULL;
400 return (-1);
401 }
402
403 *p = word;
404
405 td->td_pcb->pcb_onfault = NULL;
406 return (0);
407 }
408
409 #ifdef __powerpc64__
410 int
REMAP(suword64)411 REMAP(suword64)(volatile void *addr, int64_t word)
412 {
413 return (REMAP(suword)(addr, (long)word));
414 }
415 #endif
416
417 int
REMAP(fubyte)418 REMAP(fubyte)(volatile const void *addr)
419 {
420 struct thread *td;
421 pmap_t pm;
422 jmp_buf env;
423 u_char *p;
424 int val;
425
426 td = curthread;
427 pm = &td->td_proc->p_vmspace->vm_pmap;
428
429 td->td_pcb->pcb_onfault = &env;
430 if (setjmp(env)) {
431 td->td_pcb->pcb_onfault = NULL;
432 return (-1);
433 }
434
435 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
436 td->td_pcb->pcb_onfault = NULL;
437 return (-1);
438 }
439
440 val = *p;
441
442 td->td_pcb->pcb_onfault = NULL;
443 return (val);
444 }
445
446 int
REMAP(fuword16)447 REMAP(fuword16)(volatile const void *addr)
448 {
449 struct thread *td;
450 pmap_t pm;
451 jmp_buf env;
452 uint16_t *p, val;
453
454 td = curthread;
455 pm = &td->td_proc->p_vmspace->vm_pmap;
456
457 td->td_pcb->pcb_onfault = &env;
458 if (setjmp(env)) {
459 td->td_pcb->pcb_onfault = NULL;
460 return (-1);
461 }
462
463 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
464 td->td_pcb->pcb_onfault = NULL;
465 return (-1);
466 }
467
468 val = *p;
469
470 td->td_pcb->pcb_onfault = NULL;
471 return (val);
472 }
473
474 int
REMAP(fueword32)475 REMAP(fueword32)(volatile const void *addr, int32_t *val)
476 {
477 struct thread *td;
478 pmap_t pm;
479 jmp_buf env;
480 int32_t *p;
481
482 td = curthread;
483 pm = &td->td_proc->p_vmspace->vm_pmap;
484
485 td->td_pcb->pcb_onfault = &env;
486 if (setjmp(env)) {
487 td->td_pcb->pcb_onfault = NULL;
488 return (-1);
489 }
490
491 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
492 td->td_pcb->pcb_onfault = NULL;
493 return (-1);
494 }
495
496 *val = *p;
497
498 td->td_pcb->pcb_onfault = NULL;
499 return (0);
500 }
501
502 #ifdef __powerpc64__
503 int
REMAP(fueword64)504 REMAP(fueword64)(volatile const void *addr, int64_t *val)
505 {
506 struct thread *td;
507 pmap_t pm;
508 jmp_buf env;
509 int64_t *p;
510
511 td = curthread;
512 pm = &td->td_proc->p_vmspace->vm_pmap;
513
514 td->td_pcb->pcb_onfault = &env;
515 if (setjmp(env)) {
516 td->td_pcb->pcb_onfault = NULL;
517 return (-1);
518 }
519
520 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
521 td->td_pcb->pcb_onfault = NULL;
522 return (-1);
523 }
524
525 *val = *p;
526
527 td->td_pcb->pcb_onfault = NULL;
528 return (0);
529 }
530 #endif
531
532 int
REMAP(fueword)533 REMAP(fueword)(volatile const void *addr, long *val)
534 {
535 struct thread *td;
536 pmap_t pm;
537 jmp_buf env;
538 long *p;
539
540 td = curthread;
541 pm = &td->td_proc->p_vmspace->vm_pmap;
542
543 td->td_pcb->pcb_onfault = &env;
544 if (setjmp(env)) {
545 td->td_pcb->pcb_onfault = NULL;
546 return (-1);
547 }
548
549 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) {
550 td->td_pcb->pcb_onfault = NULL;
551 return (-1);
552 }
553
554 *val = *p;
555
556 td->td_pcb->pcb_onfault = NULL;
557 return (0);
558 }
559
560 int
REMAP(casueword32)561 REMAP(casueword32)(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp,
562 uint32_t new)
563 {
564 struct thread *td;
565 pmap_t pm;
566 jmp_buf env;
567 uint32_t *p, val;
568 int res;
569
570 td = curthread;
571 pm = &td->td_proc->p_vmspace->vm_pmap;
572
573 td->td_pcb->pcb_onfault = &env;
574 if (setjmp(env)) {
575 td->td_pcb->pcb_onfault = NULL;
576 return (-1);
577 }
578
579 if (pmap_map_user_ptr(pm, (void *)(uintptr_t)addr, (void **)&p,
580 sizeof(*p), NULL)) {
581 td->td_pcb->pcb_onfault = NULL;
582 return (-1);
583 }
584
585 res = 0;
586 __asm __volatile (
587 "lwarx %0, 0, %3\n\t" /* load old value */
588 "cmplw %4, %0\n\t" /* compare */
589 "bne 1f\n\t" /* exit if not equal */
590 "stwcx. %5, 0, %3\n\t" /* attempt to store */
591 "bne- 2f\n\t" /* if failed */
592 "b 3f\n\t" /* we've succeeded */
593 "1:\n\t"
594 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
595 "2:li %2, 1\n\t"
596 "3:\n\t"
597 : "=&r" (val), "=m" (*p), "+&r" (res)
598 : "r" (p), "r" (old), "r" (new), "m" (*p)
599 : "cr0", "memory");
600
601 td->td_pcb->pcb_onfault = NULL;
602
603 *oldvalp = val;
604 return (res);
605 }
606
607 #ifndef __powerpc64__
608 int
REMAP(casueword)609 REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new)
610 {
611
612 return (casueword32((volatile uint32_t *)addr, old,
613 (uint32_t *)oldvalp, new));
614 }
615 #else
616 int
REMAP(casueword)617 REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new)
618 {
619 struct thread *td;
620 pmap_t pm;
621 jmp_buf env;
622 u_long *p, val;
623 int res;
624
625 td = curthread;
626 pm = &td->td_proc->p_vmspace->vm_pmap;
627
628 td->td_pcb->pcb_onfault = &env;
629 if (setjmp(env)) {
630 td->td_pcb->pcb_onfault = NULL;
631 return (-1);
632 }
633
634 if (pmap_map_user_ptr(pm, (void *)(uintptr_t)addr, (void **)&p,
635 sizeof(*p), NULL)) {
636 td->td_pcb->pcb_onfault = NULL;
637 return (-1);
638 }
639
640 res = 0;
641 __asm __volatile (
642 "ldarx %0, 0, %3\n\t" /* load old value */
643 "cmpld %4, %0\n\t" /* compare */
644 "bne 1f\n\t" /* exit if not equal */
645 "stdcx. %5, 0, %3\n\t" /* attempt to store */
646 "bne- 2f\n\t" /* if failed */
647 "b 3f\n\t" /* we've succeeded */
648 "1:\n\t"
649 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
650 "2:li %2, 1\n\t"
651 "3:\n\t"
652 : "=&r" (val), "=m" (*p), "+&r" (res)
653 : "r" (p), "r" (old), "r" (new), "m" (*p)
654 : "cr0", "memory");
655
656 td->td_pcb->pcb_onfault = NULL;
657
658 *oldvalp = val;
659 return (res);
660 }
661 #endif
662