1 /*
2 * traps, exceptions, interrupts, system calls.
3 */
4 #include "u.h"
5 #include "../port/lib.h"
6 #include "mem.h"
7 #include "dat.h"
8 #include "fns.h"
9 #include "io.h"
10 #include "ureg.h"
11 #include "../port/error.h"
12
13 #include "arm.h"
14
15 #define INTREGS (VIRTIO+0xB200)
16 #define LOCALREGS (VIRTIO+IOSIZE)
17
18 typedef struct Intregs Intregs;
19 typedef struct Vctl Vctl;
20
21 enum {
22 Debug = 0,
23
24 Nvec = 8, /* # of vectors at start of lexception.s */
25 Fiqenable = 1<<7,
26
27 Localtimerint = 0x40,
28 Localmboxint = 0x50,
29 Localintpending = 0x60,
30 };
31
32 /*
33 * Layout at virtual address KZERO (double mapped at HVECTORS).
34 */
35 typedef struct Vpage0 {
36 void (*vectors[Nvec])(void);
37 u32int vtable[Nvec];
38 } Vpage0;
39
40 /*
41 * interrupt control registers
42 */
43 struct Intregs {
44 u32int ARMpending;
45 u32int GPUpending[2];
46 u32int FIQctl;
47 u32int GPUenable[2];
48 u32int ARMenable;
49 u32int GPUdisable[2];
50 u32int ARMdisable;
51 };
52
53 struct Vctl {
54 Vctl *next;
55 int irq;
56 int cpu;
57 u32int *reg;
58 u32int mask;
59 void (*f)(Ureg*, void*);
60 void *a;
61 };
62
63 static Lock vctllock;
64 static Vctl *vctl, *vfiq;
65
66 static char *trapnames[PsrMask+1] = {
67 [ PsrMusr ] "user mode",
68 [ PsrMfiq ] "fiq interrupt",
69 [ PsrMirq ] "irq interrupt",
70 [ PsrMsvc ] "svc/swi exception",
71 [ PsrMabt ] "prefetch abort/data abort",
72 [ PsrMabt+1 ] "data abort",
73 [ PsrMund ] "undefined instruction",
74 [ PsrMsys ] "sys trap",
75 };
76
77 extern int notify(Ureg*);
78
79 /*
80 * set up for exceptions
81 */
82 void
trapinit(void)83 trapinit(void)
84 {
85 Vpage0 *vpage0;
86
87 if (m->machno == 0) {
88 /* disable everything */
89 intrsoff();
90 /* set up the exception vectors */
91 vpage0 = (Vpage0*)HVECTORS;
92 memmove(vpage0->vectors, vectors, sizeof(vpage0->vectors));
93 memmove(vpage0->vtable, vtable, sizeof(vpage0->vtable));
94 cacheuwbinv();
95 l2cacheuwbinv();
96 }
97
98 /* set up the stacks for the interrupt modes */
99 setr13(PsrMfiq, (u32int*)(FIQSTKTOP));
100 setr13(PsrMirq, m->sirq);
101 setr13(PsrMabt, m->sabt);
102 setr13(PsrMund, m->sund);
103 setr13(PsrMsys, m->ssys);
104
105 coherence();
106 }
107
108 void
intrcpushutdown(void)109 intrcpushutdown(void)
110 {
111 u32int *enable;
112
113 if(soc.armlocal == 0)
114 return;
115 enable = (u32int*)(LOCALREGS + Localtimerint) + m->machno;
116 *enable = 0;
117 if(m->machno){
118 enable = (u32int*)(LOCALREGS + Localmboxint) + m->machno;
119 *enable = 1;
120 }
121 }
122
123 void
intrsoff(void)124 intrsoff(void)
125 {
126 Intregs *ip;
127 int disable;
128
129 ip = (Intregs*)INTREGS;
130 disable = ~0;
131 ip->GPUdisable[0] = disable;
132 ip->GPUdisable[1] = disable;
133 ip->ARMdisable = disable;
134 ip->FIQctl = 0;
135 }
136
137 /* called from cpu0 after other cpus are shutdown */
138 void
intrshutdown(void)139 intrshutdown(void)
140 {
141 intrsoff();
142 intrcpushutdown();
143 }
144
145 static void
intrtime(void)146 intrtime(void)
147 {
148 ulong diff;
149 ulong x;
150
151 x = perfticks();
152 diff = x - m->perf.intrts;
153 m->perf.intrts = 0;
154
155 m->perf.inintr += diff;
156 if(up == nil && m->perf.inidle > diff)
157 m->perf.inidle -= diff;
158 }
159
160
161 /*
162 * called by trap to handle irq interrupts.
163 * returns true iff a clock interrupt, thus maybe reschedule.
164 */
165 static int
irq(Ureg * ureg)166 irq(Ureg* ureg)
167 {
168 Vctl *v;
169 int clockintr;
170 int found;
171
172 m->perf.intrts = perfticks();
173 clockintr = 0;
174 found = 0;
175 for(v = vctl; v; v = v->next)
176 if(v->cpu == m->machno && (*v->reg & v->mask) != 0){
177 found = 1;
178 coherence();
179 v->f(ureg, v->a);
180 coherence();
181 if(v->irq == IRQclock || v->irq == IRQcntps || v->irq == IRQcntpns)
182 clockintr = 1;
183 }
184 if(!found)
185 m->spuriousintr++;
186 intrtime();
187 return clockintr;
188 }
189
190 /*
191 * called direct from lexception.s to handle fiq interrupt.
192 */
193 void
fiq(Ureg * ureg)194 fiq(Ureg *ureg)
195 {
196 Vctl *v;
197 int inintr;
198
199 if(m->perf.intrts)
200 inintr = 1;
201 else{
202 inintr = 0;
203 m->perf.intrts = perfticks();
204 }
205 v = vfiq;
206 if(v == nil)
207 panic("cpu%d: unexpected item in bagging area", m->machno);
208 m->intr++;
209 ureg->pc -= 4;
210 coherence();
211 v->f(ureg, v->a);
212 coherence();
213 if(!inintr)
214 intrtime();
215 }
216
217 void
irqenable(int irq,void (* f)(Ureg *,void *),void * a)218 irqenable(int irq, void (*f)(Ureg*, void*), void* a)
219 {
220 Vctl *v;
221 Intregs *ip;
222 u32int *enable;
223
224 ip = (Intregs*)INTREGS;
225 v = (Vctl*)malloc(sizeof(Vctl));
226 if(v == nil)
227 panic("irqenable: no mem");
228 v->irq = irq;
229 v->cpu = 0;
230 if(irq >= IRQlocal){
231 v->reg = (u32int*)(LOCALREGS + Localintpending) + m->machno;
232 if(irq >= IRQmbox0)
233 enable = (u32int*)(LOCALREGS + Localmboxint) + m->machno;
234 else
235 enable = (u32int*)(LOCALREGS + Localtimerint) + m->machno;
236 v->mask = 1 << (irq - IRQlocal);
237 v->cpu = m->machno;
238 }else if(irq >= IRQbasic){
239 enable = &ip->ARMenable;
240 v->reg = &ip->ARMpending;
241 v->mask = 1 << (irq - IRQbasic);
242 }else{
243 enable = &ip->GPUenable[irq/32];
244 v->reg = &ip->GPUpending[irq/32];
245 v->mask = 1 << (irq % 32);
246 }
247 v->f = f;
248 v->a = a;
249 lock(&vctllock);
250 if(irq == IRQfiq){
251 assert((ip->FIQctl & Fiqenable) == 0);
252 assert((*enable & v->mask) == 0);
253 vfiq = v;
254 ip->FIQctl = Fiqenable | irq;
255 }else{
256 v->next = vctl;
257 vctl = v;
258 if(irq >= IRQmbox0){
259 if(irq <= IRQmbox3)
260 *enable |= 1 << (irq - IRQmbox0);
261 }else if(irq >= IRQlocal)
262 *enable |= 1 << (irq - IRQlocal);
263 else
264 *enable = v->mask;
265 }
266 unlock(&vctllock);
267 }
268
269 static char *
trapname(int psr)270 trapname(int psr)
271 {
272 char *s;
273
274 s = trapnames[psr & PsrMask];
275 if(s == nil)
276 s = "unknown trap number in psr";
277 return s;
278 }
279
280 /* this is quite helpful during mmu and cache debugging */
281 static void
ckfaultstuck(uintptr va)282 ckfaultstuck(uintptr va)
283 {
284 static int cnt, lastpid;
285 static uintptr lastva;
286
287 if (va == lastva && up->pid == lastpid) {
288 ++cnt;
289 if (cnt >= 2)
290 /* fault() isn't fixing the underlying cause */
291 panic("fault: %d consecutive faults for va %#p",
292 cnt+1, va);
293 } else {
294 cnt = 0;
295 lastva = va;
296 lastpid = up->pid;
297 }
298 }
299
300 /*
301 * called by trap to handle access faults
302 */
303 static void
faultarm(Ureg * ureg,uintptr va,int user,int read)304 faultarm(Ureg *ureg, uintptr va, int user, int read)
305 {
306 int n, insyscall;
307 char buf[ERRMAX];
308
309 if(up == nil) {
310 //dumpregs(ureg);
311 panic("fault: nil up in faultarm, pc %#p accessing %#p", ureg->pc, va);
312 }
313 insyscall = up->insyscall;
314 up->insyscall = 1;
315 if (Debug)
316 ckfaultstuck(va);
317
318 n = fault(va, read);
319 if(n < 0){
320 if(!user){
321 dumpregs(ureg);
322 panic("fault: kernel accessing %#p", va);
323 }
324 /* don't dump registers; programs suicide all the time */
325 snprint(buf, sizeof buf, "sys: trap: fault %s va=%#p",
326 read? "read": "write", va);
327 postnote(up, 1, buf, NDebug);
328 }
329 up->insyscall = insyscall;
330 }
331
332 /*
333 * returns 1 if the instruction writes memory, 0 otherwise
334 */
335 int
writetomem(ulong inst)336 writetomem(ulong inst)
337 {
338 /* swap always write memory */
339 if((inst & 0x0FC00000) == 0x01000000)
340 return 1;
341
342 /* loads and stores are distinguished by bit 20 */
343 if(inst & (1<<20))
344 return 0;
345
346 return 1;
347 }
348
349 /*
350 * here on all exceptions other than syscall (SWI) and fiq
351 */
352 void
trap(Ureg * ureg)353 trap(Ureg *ureg)
354 {
355 int clockintr, user, x, rv, rem;
356 ulong inst, fsr;
357 uintptr va;
358 char buf[ERRMAX];
359
360 assert(!islo());
361 if(up != nil)
362 rem = ((char*)ureg)-up->kstack;
363 else
364 rem = ((char*)ureg)-((char*)m+sizeof(Mach));
365 if(rem < 256) {
366 iprint("trap: %d stack bytes left, up %#p ureg %#p at pc %#lux\n",
367 rem, up, ureg, ureg->pc);
368 delay(1000);
369 dumpstack();
370 panic("trap: %d stack bytes left, up %#p ureg %#p at pc %#lux",
371 rem, up, ureg, ureg->pc);
372 }
373
374 user = (ureg->psr & PsrMask) == PsrMusr;
375 if(user){
376 up->dbgreg = ureg;
377 cycles(&up->kentry);
378 }
379
380 /*
381 * All interrupts/exceptions should be resumed at ureg->pc-4,
382 * except for Data Abort which resumes at ureg->pc-8.
383 */
384 if(ureg->type == (PsrMabt+1))
385 ureg->pc -= 8;
386 else
387 ureg->pc -= 4;
388
389 clockintr = 0; /* if set, may call sched() before return */
390 switch(ureg->type){
391 default:
392 panic("unknown trap; type %#lux, psr mode %#lux pc %lux", ureg->type,
393 ureg->psr & PsrMask, ureg->pc);
394 break;
395 case PsrMirq:
396 clockintr = irq(ureg);
397 m->intr++;
398 break;
399 case PsrMabt: /* prefetch fault */
400 x = ifsrget();
401 fsr = (x>>7) & 0x8 | x & 0x7;
402 switch(fsr){
403 case 0x02: /* instruction debug event (BKPT) */
404 if(user)
405 postnote(up, 1, "sys: breakpoint", NDebug);
406 else{
407 iprint("kernel bkpt: pc %#lux inst %#ux\n",
408 ureg->pc, *(u32int*)ureg->pc);
409 panic("kernel bkpt");
410 }
411 break;
412 default:
413 faultarm(ureg, ureg->pc, user, 1);
414 break;
415 }
416 break;
417 case PsrMabt+1: /* data fault */
418 va = farget();
419 inst = *(ulong*)(ureg->pc);
420 /* bits 12 and 10 have to be concatenated with status */
421 x = fsrget();
422 fsr = (x>>7) & 0x20 | (x>>6) & 0x10 | x & 0xf;
423 switch(fsr){
424 default:
425 case 0xa: /* ? was under external abort */
426 panic("unknown data fault, 6b fsr %#lux", fsr);
427 break;
428 case 0x0:
429 panic("vector exception at %#lux", ureg->pc);
430 break;
431 case 0x1: /* alignment fault */
432 case 0x3: /* access flag fault (section) */
433 if(user){
434 snprint(buf, sizeof buf,
435 "sys: alignment: va %#p", va);
436 postnote(up, 1, buf, NDebug);
437 } else
438 panic("kernel alignment: pc %#lux va %#p", ureg->pc, va);
439 break;
440 case 0x2:
441 panic("terminal exception at %#lux", ureg->pc);
442 break;
443 case 0x4: /* icache maint fault */
444 case 0x6: /* access flag fault (page) */
445 case 0x8: /* precise external abort, non-xlat'n */
446 case 0x28:
447 case 0xc: /* l1 translation, precise ext. abort */
448 case 0x2c:
449 case 0xe: /* l2 translation, precise ext. abort */
450 case 0x2e:
451 case 0x16: /* imprecise ext. abort, non-xlt'n */
452 case 0x36:
453 panic("external abort %#lux pc %#lux addr %#p",
454 fsr, ureg->pc, va);
455 break;
456 case 0x1c: /* l1 translation, precise parity err */
457 case 0x1e: /* l2 translation, precise parity err */
458 case 0x18: /* imprecise parity or ecc err */
459 panic("translation parity error %#lux pc %#lux addr %#p",
460 fsr, ureg->pc, va);
461 break;
462 case 0x5: /* translation fault, no section entry */
463 case 0x7: /* translation fault, no page entry */
464 faultarm(ureg, va, user, !writetomem(inst));
465 break;
466 case 0x9:
467 case 0xb:
468 /* domain fault, accessing something we shouldn't */
469 if(user){
470 snprint(buf, sizeof buf,
471 "sys: access violation: va %#p", va);
472 postnote(up, 1, buf, NDebug);
473 } else
474 panic("kernel access violation: pc %#lux va %#p",
475 ureg->pc, va);
476 break;
477 case 0xd:
478 case 0xf:
479 /* permission error, copy on write or real permission error */
480 faultarm(ureg, va, user, !writetomem(inst));
481 break;
482 }
483 break;
484 case PsrMund: /* undefined instruction */
485 if(user){
486 if(seg(up, ureg->pc, 0) != nil &&
487 *(u32int*)ureg->pc == 0xD1200070)
488 postnote(up, 1, "sys: breakpoint", NDebug);
489 else{
490 /* look for floating point instructions to interpret */
491 rv = fpuemu(ureg);
492 if(rv == 0)
493 postnote(up, 1, "sys: undefined instruction", NDebug);
494 }
495 }else{
496 if (ureg->pc & 3) {
497 iprint("rounding fault pc %#lux down to word\n",
498 ureg->pc);
499 ureg->pc &= ~3;
500 }
501 iprint("undefined instruction: pc %#lux inst %#ux\n",
502 ureg->pc, *(u32int*)ureg->pc);
503 panic("undefined instruction");
504 }
505 break;
506 }
507 splhi();
508
509 /* delaysched set because we held a lock or because our quantum ended */
510 if(up && up->delaysched && clockintr){
511 sched(); /* can cause more traps */
512 splhi();
513 }
514
515 if(user){
516 if(up->procctl || up->nnote)
517 notify(ureg);
518 kexit(ureg);
519 }
520 }
521
522 int
isvalidaddr(void * v)523 isvalidaddr(void *v)
524 {
525 return (uintptr)v >= KZERO;
526 }
527
528 static void
dumplongs(char * msg,ulong * v,int n)529 dumplongs(char *msg, ulong *v, int n)
530 {
531 int i, l;
532
533 l = 0;
534 iprint("%s at %.8p: ", msg, v);
535 for(i=0; i<n; i++){
536 if(l >= 4){
537 iprint("\n %.8p: ", v);
538 l = 0;
539 }
540 if(isvalidaddr(v)){
541 iprint(" %.8lux", *v++);
542 l++;
543 }else{
544 iprint(" invalid");
545 break;
546 }
547 }
548 iprint("\n");
549 }
550
551 static void
dumpstackwithureg(Ureg * ureg)552 dumpstackwithureg(Ureg *ureg)
553 {
554 uintptr l, i, v, estack;
555 u32int *p;
556 char *s;
557
558 if((s = getconf("*nodumpstack")) != nil && strcmp(s, "0") != 0){
559 iprint("dumpstack disabled\n");
560 return;
561 }
562 iprint("ktrace /kernel/path %#.8lux %#.8lux %#.8lux # pc, sp, link\n",
563 ureg->pc, ureg->sp, ureg->r14);
564 delay(2000);
565 i = 0;
566 if(up != nil && (uintptr)&l <= (uintptr)up->kstack+KSTACK)
567 estack = (uintptr)up->kstack+KSTACK;
568 else if((uintptr)&l >= (uintptr)m->stack
569 && (uintptr)&l <= (uintptr)m+MACHSIZE)
570 estack = (uintptr)m+MACHSIZE;
571 else{
572 if(up != nil)
573 iprint("&up->kstack %#p &l %#p\n", up->kstack, &l);
574 else
575 iprint("&m %#p &l %#p\n", m, &l);
576 return;
577 }
578 for(l = (uintptr)&l; l < estack; l += sizeof(uintptr)){
579 v = *(uintptr*)l;
580 if(KTZERO < v && v < (uintptr)etext && !(v & 3)){
581 v -= sizeof(u32int); /* back up an instr */
582 p = (u32int*)v;
583 if((*p & 0x0f000000) == 0x0b000000){ /* BL instr? */
584 iprint("%#8.8lux=%#8.8lux ", l, v);
585 i++;
586 }
587 }
588 if(i == 4){
589 i = 0;
590 iprint("\n");
591 }
592 }
593 if(i)
594 iprint("\n");
595 }
596
597 /*
598 * Fill in enough of Ureg to get a stack trace, and call a function.
599 * Used by debugging interface rdb.
600 */
601
602 static void
getpcsp(ulong * pc,ulong * sp)603 getpcsp(ulong *pc, ulong *sp)
604 {
605 *pc = getcallerpc(&pc);
606 *sp = (ulong)&pc-4;
607 }
608
609 void
callwithureg(void (* fn)(Ureg *))610 callwithureg(void (*fn)(Ureg*))
611 {
612 Ureg ureg;
613
614 getpcsp((ulong*)&ureg.pc, (ulong*)&ureg.sp);
615 ureg.r14 = getcallerpc(&fn);
616 fn(&ureg);
617 }
618
619 void
dumpstack(void)620 dumpstack(void)
621 {
622 callwithureg(dumpstackwithureg);
623 }
624
625 void
dumpregs(Ureg * ureg)626 dumpregs(Ureg* ureg)
627 {
628 int s;
629
630 if (ureg == nil) {
631 iprint("trap: no user process\n");
632 return;
633 }
634 s = splhi();
635 iprint("trap: %s", trapname(ureg->type));
636 if(ureg != nil && (ureg->psr & PsrMask) != PsrMsvc)
637 iprint(" in %s", trapname(ureg->psr));
638 iprint("\n");
639 iprint("psr %8.8lux type %2.2lux pc %8.8lux link %8.8lux\n",
640 ureg->psr, ureg->type, ureg->pc, ureg->link);
641 iprint("R14 %8.8lux R13 %8.8lux R12 %8.8lux R11 %8.8lux R10 %8.8lux\n",
642 ureg->r14, ureg->r13, ureg->r12, ureg->r11, ureg->r10);
643 iprint("R9 %8.8lux R8 %8.8lux R7 %8.8lux R6 %8.8lux R5 %8.8lux\n",
644 ureg->r9, ureg->r8, ureg->r7, ureg->r6, ureg->r5);
645 iprint("R4 %8.8lux R3 %8.8lux R2 %8.8lux R1 %8.8lux R0 %8.8lux\n",
646 ureg->r4, ureg->r3, ureg->r2, ureg->r1, ureg->r0);
647 iprint("stack is at %#p\n", ureg);
648 iprint("pc %#lux link %#lux\n", ureg->pc, ureg->link);
649
650 if(up)
651 iprint("user stack: %#p-%#p\n", up->kstack, up->kstack+KSTACK-4);
652 else
653 iprint("kernel stack: %8.8lux-%8.8lux\n",
654 (ulong)(m+1), (ulong)m+BY2PG-4);
655 dumplongs("stack", (ulong *)(ureg + 1), 16);
656 delay(2000);
657 dumpstack();
658 splx(s);
659 }
660