xref: /netbsd-src/sys/arch/powerpc/ibm4xx/trap.c (revision 68fa58437753598de948829082f591c269b48777)
1 /*	$NetBSD: trap.c,v 1.102 2023/10/05 19:41:05 ad Exp $	*/
2 
3 /*
4  * Copyright 2001 Wasabi Systems, Inc.
5  * All rights reserved.
6  *
7  * Written by Eduardo Horvath and Simon Burge for Wasabi Systems, Inc.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  * 3. All advertising materials mentioning features or use of this software
18  *    must display the following acknowledgement:
19  *      This product includes software developed for the NetBSD Project by
20  *      Wasabi Systems, Inc.
21  * 4. The name of Wasabi Systems, Inc. may not be used to endorse
22  *    or promote products derived from this software without specific prior
23  *    written permission.
24  *
25  * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
26  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
27  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
29  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35  * POSSIBILITY OF SUCH DAMAGE.
36  */
37 
38 /*
39  * Copyright (C) 1995, 1996 Wolfgang Solfrank.
40  * Copyright (C) 1995, 1996 TooLs GmbH.
41  * All rights reserved.
42  *
43  * Redistribution and use in source and binary forms, with or without
44  * modification, are permitted provided that the following conditions
45  * are met:
46  * 1. Redistributions of source code must retain the above copyright
47  *    notice, this list of conditions and the following disclaimer.
48  * 2. Redistributions in binary form must reproduce the above copyright
49  *    notice, this list of conditions and the following disclaimer in the
50  *    documentation and/or other materials provided with the distribution.
51  * 3. All advertising materials mentioning features or use of this software
52  *    must display the following acknowledgement:
53  *	This product includes software developed by TooLs GmbH.
54  * 4. The name of TooLs GmbH may not be used to endorse or promote products
55  *    derived from this software without specific prior written permission.
56  *
57  * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
58  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
59  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
60  * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
61  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
62  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
63  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
64  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
65  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
66  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
67  */
68 
69 #define	__UFETCHSTORE_PRIVATE
70 
71 #include <sys/cdefs.h>
72 __KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.102 2023/10/05 19:41:05 ad Exp $");
73 
74 #ifdef _KERNEL_OPT
75 #include "opt_ddb.h"
76 #include "opt_kgdb.h"
77 #include "opt_ppcarch.h"
78 #include "opt_ppcopts.h"
79 #endif
80 
81 #include <sys/param.h>
82 #include <sys/cpu.h>
83 #include <sys/kauth.h>
84 #include <sys/proc.h>
85 #include <sys/ptrace.h>
86 #include <sys/reboot.h>
87 #include <sys/syscall.h>
88 #include <sys/systm.h>
89 
90 #if defined(KGDB)
91 #include <sys/kgdb.h>
92 #endif
93 
94 #include <uvm/uvm_extern.h>
95 
96 #include <dev/cons.h>
97 
98 #include <machine/fpu.h>
99 #include <machine/frame.h>
100 #include <machine/pcb.h>
101 #include <machine/psl.h>
102 #include <machine/trap.h>
103 
104 #include <powerpc/db_machdep.h>
105 #include <powerpc/spr.h>
106 #include <powerpc/userret.h>
107 
108 #include <powerpc/ibm4xx/cpu.h>
109 #include <powerpc/ibm4xx/pmap.h>
110 #include <powerpc/ibm4xx/spr.h>
111 #include <powerpc/ibm4xx/tlb.h>
112 
113 #include <powerpc/fpu/fpu_extern.h>
114 
115 /* These definitions should probably be somewhere else			XXX */
116 #define	FIRSTARG	3		/* first argument is in reg 3 */
117 #define	NARGREG		8		/* 8 args are in registers */
118 #define	MOREARGS(sp)	((void *)((int)(sp) + 8)) /* more args go here */
119 
120 void trap(struct trapframe *);	/* Called from locore / trap_subr */
121 #if 0
122 /* Not currently used nor exposed externally in any header file */
123 int badaddr(void *, size_t);
124 int badaddr_read(void *, size_t, int *);
125 #endif
126 int ctx_setup(int, int);
127 
128 #ifndef PPC_NO_UNALIGNED
129 static bool fix_unaligned(struct trapframe *, ksiginfo_t *);
130 #endif
131 
132 #ifdef DEBUG
133 #define TDB_ALL	0x1
134 int trapdebug = /* TDB_ALL */ 0;
135 #define	DBPRINTF(x, y)	if (trapdebug & (x)) printf y
136 #else
137 #define DBPRINTF(x, y)
138 #endif
139 
140 void
trap(struct trapframe * tf)141 trap(struct trapframe *tf)
142 {
143 	struct lwp *l = curlwp;
144 	struct proc *p = l->l_proc;
145 	struct pcb *pcb;
146 	int type = tf->tf_exc;
147 	int ftype, rv;
148 	ksiginfo_t ksi;
149 
150 	KASSERT(l->l_stat == LSONPROC);
151 
152 	if (tf->tf_srr1 & PSL_PR) {
153 		type |= EXC_USER;
154 	}
155 
156 	ftype = VM_PROT_READ;
157 
158 	DBPRINTF(TDB_ALL, ("trap(%x) at %lx from frame %p &frame %p\n",
159 	    type, tf->tf_srr0, tf, &tf));
160 
161 	switch (type) {
162 	case EXC_DEBUG|EXC_USER:
163 		/* We don't use hardware breakpoints for userland. */
164 		goto brain_damage;
165 
166 	case EXC_TRC|EXC_USER:
167 		KSI_INIT_TRAP(&ksi);
168 		ksi.ksi_signo = SIGTRAP;
169 		ksi.ksi_trap = EXC_TRC;
170 		ksi.ksi_addr = (void *)tf->tf_srr0;
171 		trapsignal(l, &ksi);
172 		break;
173 
174 	case EXC_DSI:
175 		/* FALLTHROUGH */
176 	case EXC_DTMISS:
177 		{
178 			struct vm_map *map;
179 			vaddr_t va;
180 			struct faultbuf *fb;
181 
182 			pcb = lwp_getpcb(l);
183 			fb = pcb->pcb_onfault;
184 
185 			if (curcpu()->ci_idepth >= 0) {
186 				rv = EFAULT;
187 				goto out;
188 			}
189 
190 			va = tf->tf_dear;
191 			if (tf->tf_pid == KERNEL_PID) {
192 				map = kernel_map;
193 			} else {
194 				map = &p->p_vmspace->vm_map;
195 			}
196 
197 			if (tf->tf_esr & (ESR_DST|ESR_DIZ))
198 				ftype = VM_PROT_WRITE;
199 
200 			DBPRINTF(TDB_ALL,
201 			    ("trap(EXC_DSI) at %lx %s fault on %p esr %x\n",
202 			    tf->tf_srr0,
203 			    (ftype & VM_PROT_WRITE) ? "write" : "read",
204 			    (void *)va, tf->tf_esr));
205 
206 			pcb->pcb_onfault = NULL;
207 			rv = uvm_fault(map, trunc_page(va), ftype);
208 			pcb->pcb_onfault = fb;
209 			if (rv == 0)
210 				return;
211 out:
212 			if (fb != NULL) {
213 				tf->tf_pid = KERNEL_PID;
214 				tf->tf_srr0 = fb->fb_pc;
215 				tf->tf_srr1 |= PSL_IR; /* Re-enable IMMU */
216 				tf->tf_cr = fb->fb_cr;
217 				tf->tf_fixreg[1] = fb->fb_sp;
218 				tf->tf_fixreg[2] = fb->fb_r2;
219 				tf->tf_fixreg[3] = rv;
220 				memcpy(&tf->tf_fixreg[13], fb->fb_fixreg,
221 				    sizeof(fb->fb_fixreg));
222 				return;
223 			}
224 		}
225 		goto brain_damage;
226 
227 	case EXC_DSI|EXC_USER:
228 		/* FALLTHROUGH */
229 	case EXC_DTMISS|EXC_USER:
230 		if (tf->tf_esr & (ESR_DST|ESR_DIZ))
231 			ftype = VM_PROT_WRITE;
232 
233 		DBPRINTF(TDB_ALL,
234 		    ("trap(EXC_DSI|EXC_USER) at %lx %s fault on %lx %x\n",
235 		    tf->tf_srr0, (ftype & VM_PROT_WRITE) ? "write" : "read",
236 		    tf->tf_dear, tf->tf_esr));
237 		KASSERT(l == curlwp && (l->l_stat == LSONPROC));
238 //		KASSERT(curpcb->pcb_onfault == NULL);
239 		rv = uvm_fault(&p->p_vmspace->vm_map, trunc_page(tf->tf_dear),
240 		    ftype);
241 		if (rv == 0) {
242 			break;
243 		}
244 		KSI_INIT_TRAP(&ksi);
245 		ksi.ksi_trap = EXC_DSI;
246 		ksi.ksi_addr = (void *)tf->tf_dear;
247 vm_signal:
248 		switch (rv) {
249 		case EINVAL:
250 			ksi.ksi_signo = SIGBUS;
251 			ksi.ksi_code = BUS_ADRERR;
252 			break;
253 		case EACCES:
254 			ksi.ksi_signo = SIGSEGV;
255 			ksi.ksi_code = SEGV_ACCERR;
256 			break;
257 		case ENOMEM:
258 			ksi.ksi_signo = SIGKILL;
259 			printf("UVM: pid %d.%d (%s), uid %d killed: "
260 			       "out of swap\n", p->p_pid, l->l_lid, p->p_comm,
261 			       l->l_cred ? kauth_cred_geteuid(l->l_cred) : -1);
262 			break;
263 		default:
264 			ksi.ksi_signo = SIGSEGV;
265 			ksi.ksi_code = SEGV_MAPERR;
266 			break;
267 		}
268 		trapsignal(l, &ksi);
269 		break;
270 
271 	case EXC_ITMISS|EXC_USER:
272 	case EXC_ISI|EXC_USER:
273 		ftype = VM_PROT_EXECUTE;
274 		DBPRINTF(TDB_ALL,
275 		    ("trap(EXC_ISI|EXC_USER) at %lx execute fault tf %p\n",
276 		    tf->tf_srr0, tf));
277 //		KASSERT(curpcb->pcb_onfault == NULL);
278 		rv = uvm_fault(&p->p_vmspace->vm_map, trunc_page(tf->tf_srr0),
279 		    ftype);
280 		if (rv == 0) {
281 			break;
282 		}
283 isi:
284 		KSI_INIT_TRAP(&ksi);
285 		ksi.ksi_trap = EXC_ISI;
286 		ksi.ksi_addr = (void *)tf->tf_srr0;
287 		goto vm_signal;
288 		break;
289 
290 	case EXC_AST|EXC_USER:
291 		cpu_ast(l, curcpu());
292 		break;
293 
294 	case EXC_ALI|EXC_USER:
295 		if (fix_unaligned(tf, &ksi))
296 			trapsignal(l, &ksi);
297 		break;
298 
299 	case EXC_PGM|EXC_USER:
300 		curcpu()->ci_data.cpu_ntrap++;
301 
302 		KSI_INIT_TRAP(&ksi);
303 		ksi.ksi_trap = EXC_PGM;
304 		ksi.ksi_addr = (void *)tf->tf_srr0;
305 
306 		if (tf->tf_esr & ESR_PTR) {
307 			vaddr_t va;
308 sigtrap:
309 			va = (vaddr_t)tf->tf_srr0;
310 			/*
311 		 	 * Restore original instruction and clear BP.
312 		 	 */
313 			if (p->p_md.md_ss_addr[0] == va ||
314 			    p->p_md.md_ss_addr[1] == va) {
315 				rv = ppc_sstep(l, 0);
316 				if (rv != 0)
317 					goto vm_signal;
318 				ksi.ksi_code = TRAP_TRACE;
319 			} else
320 				ksi.ksi_code = TRAP_BRKPT;
321 			if (p->p_raslist != NULL &&
322 			    ras_lookup(p, (void *)va) != (void *)-1) {
323 				tf->tf_srr0 += (ksi.ksi_code == TRAP_TRACE) ?
324 				    0 : 4;
325 				break;
326 			}
327 			ksi.ksi_signo = SIGTRAP;
328 		} else if (tf->tf_esr & ESR_PPR) {
329 			uint32_t opcode;
330 
331 			rv = copyin((void *)tf->tf_srr0, &opcode,
332 			    sizeof(opcode));
333 			if (rv)
334 				goto isi;
335 			if (emulate_mxmsr(l, tf, opcode)) {
336 				tf->tf_srr0 += 4;
337 				break;
338 			}
339 
340 			ksi.ksi_code = ILL_PRVOPC;
341 			ksi.ksi_signo = SIGILL;
342 		} else {
343 			pcb = lwp_getpcb(l);
344 
345 			if (__predict_false(!fpu_used_p(l))) {
346 				memset(&pcb->pcb_fpu, 0, sizeof(pcb->pcb_fpu));
347 				fpu_mark_used(l);
348 			}
349 
350 			if (fpu_emulate(tf, &pcb->pcb_fpu, &ksi)) {
351 				if (ksi.ksi_signo == 0)	/* was emulated */
352 					break;
353 				else if (ksi.ksi_signo == SIGTRAP)
354 					goto sigtrap;	/* XXX H/W bug? */
355 			} else {
356 				ksi.ksi_code = ILL_ILLOPC;
357 				ksi.ksi_signo = SIGILL;
358 			}
359 		}
360 
361 		trapsignal(l, &ksi);
362 		break;
363 
364 	case EXC_MCHK:
365 		{
366 			struct faultbuf *fb;
367 
368 			pcb = lwp_getpcb(l);
369 			if ((fb = pcb->pcb_onfault) != NULL) {
370 				tf->tf_pid = KERNEL_PID;
371 				tf->tf_srr0 = fb->fb_pc;
372 				tf->tf_srr1 |= PSL_IR; /* Re-enable IMMU */
373 				tf->tf_fixreg[1] = fb->fb_sp;
374 				tf->tf_fixreg[2] = fb->fb_r2;
375 				tf->tf_fixreg[3] = 1; /* Return TRUE */
376 				tf->tf_cr = fb->fb_cr;
377 				memcpy(&tf->tf_fixreg[13], fb->fb_fixreg,
378 				    sizeof(fb->fb_fixreg));
379 				return;
380 			}
381 		}
382 		goto brain_damage;
383 
384 	default:
385 brain_damage:
386 		printf("trap type 0x%x at 0x%lx\n", type, tf->tf_srr0);
387 #if defined(DDB) || defined(KGDB)
388 		if (kdb_trap(type, tf))
389 			return;
390 #endif
391 #ifdef TRAP_PANICWAIT
392 		printf("Press a key to panic.\n");
393 		cngetc();
394 #endif
395 		panic("trap");
396 	}
397 
398 	/* Invoke powerpc userret code */
399 	userret(l, tf);
400 }
401 
402 int
ctx_setup(int ctx,int srr1)403 ctx_setup(int ctx, int srr1)
404 {
405 	volatile struct pmap *pm;
406 
407 	/* Update PID if we're returning to user mode. */
408 	if (srr1 & PSL_PR) {
409 		pm = curproc->p_vmspace->vm_map.pmap;
410 		if (!pm->pm_ctx) {
411 			ctx_alloc(__UNVOLATILE(pm));
412 		}
413 		ctx = pm->pm_ctx;
414 	}
415 	else if (!ctx) {
416 		ctx = KERNEL_PID;
417 	}
418 	return (ctx);
419 }
420 
421 /*
422  * Used by copyin()/copyout()
423  */
424 extern vaddr_t vmaprange(struct proc *, vaddr_t, vsize_t, int);
425 extern void vunmaprange(vaddr_t, vsize_t);
426 static int bigcopyin(const void *, void *, size_t );
427 static int bigcopyout(const void *, void *, size_t );
428 
429 #ifdef __clang__
430 #pragma clang optimize off
431 #endif
432 int
copyin(const void * uaddr,void * kaddr,size_t len)433 copyin(const void *uaddr, void *kaddr, size_t len)
434 {
435 	struct pmap *pm = curproc->p_vmspace->vm_map.pmap;
436 	int rv, msr, pid, tmp, ctx;
437 	struct faultbuf env;
438 
439 	/* For bigger buffers use the faster copy */
440 	if (len > 1024)
441 		return (bigcopyin(uaddr, kaddr, len));
442 
443 	if ((rv = setfault(&env))) {
444 		curpcb->pcb_onfault = NULL;
445 		return rv;
446 	}
447 
448 	if (!(ctx = pm->pm_ctx)) {
449 		/* No context -- assign it one */
450 		ctx_alloc(pm);
451 		ctx = pm->pm_ctx;
452 	}
453 
454 	__asm volatile (
455 		"mfmsr	%[msr];"		/* Save MSR */
456 		"li	%[tmp],0x20;"		/* Disable IMMU */
457 		"andc	%[tmp],%[msr],%[tmp];"
458 		"mtmsr	%[tmp];"
459 		"isync;"
460 		MFPID(%[pid])			/* Save old PID */
461 
462 		"srwi.	%[tmp],%[len],0x2;"	/* How many words? */
463 		"beq-	2f;"			/* No words. Go do bytes */
464 		"mtctr	%[tmp];"
465 
466 	"1:"	MTPID(%[ctx])
467 		"isync;"
468 #ifdef PPC_IBM403
469 		"lswi	%[tmp],%[uaddr],4;"	/* Load user word */
470 #else
471 		"lwz	%[tmp],0(%[uaddr]);"
472 #endif
473 		"addi	%[uaddr],%[uaddr],0x4;"	/* next uaddr word */
474 		"sync;"
475 
476 		MTPID(%[pid])
477 		"isync;"
478 #ifdef PPC_IBM403
479 		"stswi	%[tmp],%[kaddr],4;"	/* Store kernel word */
480 #else
481 		"stw	%[tmp],0(%[kaddr]);"
482 #endif
483 		"addi	%[kaddr],%[kaddr],0x4;"	/* next kaddr word */
484 		"sync;"
485 		"bdnz	1b;"			/* repeat */
486 
487 	"2:"	"andi.	%[tmp],%[len],0x3;"	/* How many remaining bytes? */
488 		"beq	10f;"
489 		"mtxer	%[tmp];"
490 
491 		MTPID(%[ctx])
492 		"isync;"
493 		"lswx	%[tmp],0,%[uaddr];"	/* Load user bytes */
494 		"sync;"
495 
496 		MTPID(%[pid])
497 		"isync;"
498 		"stswx	%[tmp],0,%[kaddr];"	/* Store kernel bytes */
499 		"sync;"
500 
501 	"10:"	"mtmsr	%[msr];"		/* Restore MSR */
502 		"isync;"
503 
504 		: [msr] "=&r" (msr), [pid] "=&r" (pid), [tmp] "=&r" (tmp)
505 		: [uaddr] "b" (uaddr), [kaddr] "b" (kaddr),
506 		  [ctx] "r" (ctx), [len] "r" (len)
507 		: "cr0", "ctr", "xer");
508 
509 	curpcb->pcb_onfault = NULL;
510 	return 0;
511 }
512 #ifdef __clang__
513 #pragma clang optimize on
514 #endif
515 
516 static int
bigcopyin(const void * uaddr,void * kaddr,size_t len)517 bigcopyin(const void *uaddr, void *kaddr, size_t len)
518 {
519 	const char *up;
520 	char *kp = kaddr;
521 	struct lwp *l = curlwp;
522 	struct proc *p;
523 	struct faultbuf env;
524 	int error;
525 
526 	p = l->l_proc;
527 
528 	/*
529 	 * Stolen from physio():
530 	 */
531 	error = uvm_vslock(p->p_vmspace, __UNCONST(uaddr), len, VM_PROT_READ);
532 	if (error) {
533 		return error;
534 	}
535 	up = (char *)vmaprange(p, (vaddr_t)uaddr, len, VM_PROT_READ);
536 
537 	if ((error = setfault(&env)) == 0) {
538 		memcpy(kp, up, len);
539 	}
540 
541 	curpcb->pcb_onfault = NULL;
542 	vunmaprange((vaddr_t)up, len);
543 	uvm_vsunlock(p->p_vmspace, __UNCONST(uaddr), len);
544 
545 	return error;
546 }
547 
548 #ifdef __clang__
549 #pragma clang optimize off
550 #endif
551 int
copyout(const void * kaddr,void * uaddr,size_t len)552 copyout(const void *kaddr, void *uaddr, size_t len)
553 {
554 	struct pmap *pm = curproc->p_vmspace->vm_map.pmap;
555 	int rv, msr, pid, tmp, ctx;
556 	struct faultbuf env;
557 
558 	/* For big copies use more efficient routine */
559 	if (len > 1024)
560 		return (bigcopyout(kaddr, uaddr, len));
561 
562 	if ((rv = setfault(&env))) {
563 		curpcb->pcb_onfault = NULL;
564 		return rv;
565 	}
566 
567 	if (!(ctx = pm->pm_ctx)) {
568 		/* No context -- assign it one */
569 		ctx_alloc(pm);
570 		ctx = pm->pm_ctx;
571 	}
572 
573 	__asm volatile (
574 		"mfmsr	%[msr];"		/* Save MSR */
575 		"li	%[tmp],0x20;"		/* Disable IMMU */
576 		"andc	%[tmp],%[msr],%[tmp];"
577 		"mtmsr	%[tmp];"
578 		"isync;"
579 		MFPID(%[pid])			/* Save old PID */
580 
581 		"srwi.	%[tmp],%[len],0x2;"	/* How many words? */
582 		"beq-	2f;"			/* No words. Go do bytes */
583 		"mtctr	%[tmp];"
584 
585 	"1:"
586 #ifdef PPC_IBM403
587 		"lswi	%[tmp],%[kaddr],4;"	/* Load kernel word */
588 #else
589 		"lwz	%[tmp],0(%[kaddr]);"
590 #endif
591 		"addi	%[kaddr],%[kaddr],0x4;"	/* next kaddr word */
592 		"sync;"
593 
594 		MTPID(%[ctx])
595 		"isync;"
596 #ifdef PPC_IBM403
597 		"stswi	%[tmp],%[uaddr],4;"	/* Store user word */
598 #else
599 		"stw	%[tmp],0(%[uaddr]);"
600 #endif
601 		"addi	%[uaddr],%[uaddr],0x4;"	/* next uaddr word */
602 		"sync;"
603 
604 		MTPID(%[pid])
605 		"isync;"
606 		"bdnz	1b;"			/* repeat */
607 
608 	"2:"	"andi.	%[tmp],%[len],0x3;"	/* How many remaining bytes? */
609 		"beq	10f;"
610 		"mtxer	%[tmp];"
611 
612 		"lswx	%[tmp],0,%[kaddr];"	/* Load kernel bytes */
613 		"sync;"
614 
615 		MTPID(%[ctx])
616 		"isync;"
617 		"stswx	%[tmp],0,%[uaddr];"	/* Store user bytes */
618 		"sync;"
619 
620 		MTPID(%[pid])			/* Restore PID and MSR */
621 	"10:"	"mtmsr	%[msr];"
622 		"isync;"
623 
624 		: [msr] "=&r" (msr), [pid] "=&r" (pid), [tmp] "=&r" (tmp)
625 		: [uaddr] "b" (uaddr), [kaddr] "b" (kaddr),
626 		  [ctx] "r" (ctx), [len] "r" (len)
627 		: "cr0", "ctr", "xer");
628 
629 	curpcb->pcb_onfault = NULL;
630 	return 0;
631 }
632 #ifdef __clang__
633 #pragma clang optimize on
634 #endif
635 
636 static int
bigcopyout(const void * kaddr,void * uaddr,size_t len)637 bigcopyout(const void *kaddr, void *uaddr, size_t len)
638 {
639 	char *up;
640 	const char *kp = (const char *)kaddr;
641 	struct lwp *l = curlwp;
642 	struct proc *p;
643 	struct faultbuf env;
644 	int error;
645 
646 	p = l->l_proc;
647 
648 	/*
649 	 * Stolen from physio():
650 	 */
651 	error = uvm_vslock(p->p_vmspace, uaddr, len, VM_PROT_WRITE);
652 	if (error) {
653 		return error;
654 	}
655 	up = (char *)vmaprange(p, (vaddr_t)uaddr, len,
656 	    VM_PROT_READ | VM_PROT_WRITE);
657 
658 	if ((error = setfault(&env)) == 0) {
659 		memcpy(up, kp, len);
660 	}
661 
662 	curpcb->pcb_onfault = NULL;
663 	vunmaprange((vaddr_t)up, len);
664 	uvm_vsunlock(p->p_vmspace, uaddr, len);
665 
666 	return error;
667 }
668 
669 /*
670  * kcopy(const void *src, void *dst, size_t len);
671  *
672  * Copy len bytes from src to dst, aborting if we encounter a fatal
673  * page fault.
674  *
675  * kcopy() _must_ save and restore the old fault handler since it is
676  * called by uiomove(), which may be in the path of servicing a non-fatal
677  * page fault.
678  */
679 int
kcopy(const void * src,void * dst,size_t len)680 kcopy(const void *src, void *dst, size_t len)
681 {
682 	struct faultbuf env, *oldfault;
683 	int rv;
684 
685 	oldfault = curpcb->pcb_onfault;
686 	if ((rv = setfault(&env))) {
687 		curpcb->pcb_onfault = oldfault;
688 		return rv;
689 	}
690 
691 	memcpy(dst, src, len);
692 
693 	curpcb->pcb_onfault = oldfault;
694 	return 0;
695 }
696 
697 #if 0
698 int
699 badaddr(void *addr, size_t size)
700 {
701 
702 	return badaddr_read(addr, size, NULL);
703 }
704 
705 int
706 badaddr_read(void *addr, size_t size, int *rptr)
707 {
708 	struct faultbuf env;
709 	int x;
710 
711 	/* Get rid of any stale machine checks that have been waiting.  */
712 	__asm volatile ("sync; isync");
713 
714 	if (setfault(&env)) {
715 		curpcb->pcb_onfault = NULL;
716 		__asm volatile ("sync");
717 		return 1;
718 	}
719 
720 	__asm volatile ("sync");
721 
722 	switch (size) {
723 	case 1:
724 		x = *(volatile int8_t *)addr;
725 		break;
726 	case 2:
727 		x = *(volatile int16_t *)addr;
728 		break;
729 	case 4:
730 		x = *(volatile int32_t *)addr;
731 		break;
732 	default:
733 		panic("badaddr: invalid size (%d)", size);
734 	}
735 
736 	/* Make sure we took the machine check, if we caused one. */
737 	__asm volatile ("sync; isync");
738 
739 	curpcb->pcb_onfault = NULL;
740 	__asm volatile ("sync");	/* To be sure. */
741 
742 	/* Use the value to avoid reorder. */
743 	if (rptr)
744 		*rptr = x;
745 
746 	return 0;
747 }
748 #endif
749 
750 #ifndef PPC_NO_UNALIGNED
751 static bool
fix_unaligned(struct trapframe * tf,ksiginfo_t * ksi)752 fix_unaligned(struct trapframe *tf, ksiginfo_t *ksi)
753 {
754 
755 	KSI_INIT_TRAP(ksi);
756 	ksi->ksi_signo = SIGBUS;
757 	ksi->ksi_trap = EXC_ALI;
758 	ksi->ksi_addr = (void *)tf->tf_dear;
759 	return true;
760 }
761 #endif
762 
763 /*
764  * XXX Extremely lame implementations of _ufetch_* / _ustore_*.  IBM 4xx
765  * experts should make versions that are good.
766  */
767 
768 #define UFETCH(sz)							\
769 int									\
770 _ufetch_ ## sz(const uint ## sz ## _t *uaddr, uint ## sz ## _t *valp)	\
771 {									\
772 	return copyin(uaddr, valp, sizeof(*valp));			\
773 }
774 
775 UFETCH(8)
776 UFETCH(16)
777 UFETCH(32)
778 
779 #define USTORE(sz)							\
780 int									\
781 _ustore_ ## sz(uint ## sz ## _t *uaddr, uint ## sz ## _t val)		\
782 {									\
783 	return copyout(&val, uaddr, sizeof(val));			\
784 }
785 
786 USTORE(8)
787 USTORE(16)
788 USTORE(32)
789