1 /*
2 * cortex-a clocks; excludes tegra 2 SoC clocks
3 *
4 * cortex-a processors include private `global' and local timers
5 * at soc.scu + 0x200 (global) and + 0x600 (local).
6 * the global timer is a single count-up timer shared by all cores
7 * but with per-cpu comparator and auto-increment registers.
8 * a local count-down timer can be used as a watchdog.
9 *
10 * v7 arch provides a 32-bit count-up cycle counter (at about 1GHz in our case)
11 * but it's unsuitable as our source of fastticks, because it stops advancing
12 * when the cpu is suspended by WFI.
13 */
14 #include "u.h"
15 #include "../port/lib.h"
16 #include "mem.h"
17 #include "dat.h"
18 #include "fns.h"
19 #include "arm.h"
20
21 enum {
22 Debug = 0,
23
24 Basetickfreq = Mhz, /* soc.µs rate in Hz */
25 /* the local timers seem to run at half the expected rate */
26 Clockfreqbase = 250*Mhz / 2, /* private timer rate (PERIPHCLK/2) */
27 Tcycles = Clockfreqbase / HZ, /* cycles per clock tick */
28
29 MinPeriod = Tcycles / 100,
30 MaxPeriod = Tcycles,
31
32 Dogtimeout = Dogsectimeout * Clockfreqbase,
33 };
34
35 typedef struct Ltimer Ltimer;
36 typedef struct Pglbtmr Pglbtmr;
37 typedef struct Ploctmr Ploctmr;
38
39 /*
40 * cortex-a private-intr local timer registers. all cpus see their
41 * own local timers at the same base address.
42 */
43 struct Ltimer {
44 ulong load; /* new value + 1 */
45 ulong cnt; /* counts down */
46 ulong ctl;
47 ulong isr;
48
49 /* watchdog only */
50 ulong wdrst;
51 ulong wddis; /* wo */
52
53 ulong _pad0[2];
54 };
55 struct Ploctmr {
56 Ltimer loc;
57 Ltimer wd;
58 };
59
60 enum {
61 /* ctl bits */
62 Tmrena = 1<<0, /* timer enabled */
63 Wdogena = Tmrena, /* watchdog enabled */
64 Xreload = 1<<1, /* reload on intr; periodic interrupts */
65 Tintena = 1<<2, /* enable irq 29 at cnt==0 (30 for watchdog) */
66 Wdog = 1<<3, /* watchdog, not timer, mode */
67 Xsclrshift = 8,
68 Xsclrmask = MASK(8),
69
70 /* isr bits */
71 Xisrclk = 1<<0, /* write to clear */
72
73 /* wdrst bits */
74 Wdrst = 1<<0,
75
76 /* wddis values */
77 Wdon = 1,
78 Wdoff1 = 0x12345678, /* send these two to switch to timer mode */
79 Wdoff2 = 0x87654321,
80 };
81
82 /* cortex-a private-intr globl timer registers */
83 struct Pglbtmr {
84 ulong cnt[2]; /* counts up; little-endian uvlong */
85 ulong ctl;
86 ulong isr;
87 ulong cmp[2]; /* little-endian uvlong */
88 ulong inc;
89 };
90
91 enum {
92 /* unique ctl bits (otherwise see X* above) */
93 Gcmp = 1<<1,
94 // Gtintena= 1<<2, /* enable irq 27 */
95 Gincr = 1<<3,
96 };
97
98 /*
99 * until 5[cl] inline vlong ops, avoid them where possible,
100 * they are currently slow function calls.
101 */
102 typedef union Vlong Vlong;
103 union Vlong {
104 uvlong uvl;
105 struct { /* little-endian */
106 ulong low;
107 ulong high;
108 };
109 };
110
111 static int fired;
112 static int ticking[MAXMACH];
113
114 /* no lock is needed to update our local timer. splhi keeps it tight. */
115 static void
setltimer(Ltimer * tn,ulong ticks)116 setltimer(Ltimer *tn, ulong ticks)
117 {
118 int s;
119
120 assert(ticks <= Clockfreqbase);
121 s = splhi();
122 tn->load = ticks - 1;
123 coherence();
124 tn->ctl = Tmrena | Tintena | Xreload;
125 coherence();
126 splx(s);
127 }
128
129 static void
ckstuck(int cpu,long myticks,long histicks)130 ckstuck(int cpu, long myticks, long histicks)
131 {
132 if (labs(histicks - myticks) > HZ) {
133 // iprint("cpu%d: clock ticks %ld (vs myticks %ld cpu0 %ld); "
134 // "apparently stopped\n",
135 // cpu, histicks, myticks, MACHP(0)->ticks);
136 if (!ticking[cpu])
137 panic("cpu%d: clock not interrupting", cpu);
138 }
139 }
140
141 static void
mpclocksanity(void)142 mpclocksanity(void)
143 {
144 int cpu, mycpu;
145 long myticks, histicks;
146
147 if (conf.nmach <= 1 || active.exiting || navailcpus == 0)
148 return;
149
150 mycpu = m->machno;
151 myticks = m->ticks;
152 if (myticks == HZ)
153 ticking[mycpu] = 1;
154
155 if (myticks < 5*HZ)
156 return;
157
158 for (cpu = 0; cpu < navailcpus; cpu++) {
159 if (cpu == mycpu)
160 continue;
161 histicks = MACHP(cpu)->ticks;
162 if (myticks == 5*HZ || histicks > 1)
163 ckstuck(cpu, myticks, histicks);
164 }
165 }
166
167 static void
clockintr(Ureg * ureg,void * arg)168 clockintr(Ureg* ureg, void *arg)
169 {
170 Ltimer *wd, *tn;
171 Ploctmr *lt;
172
173 lt = (Ploctmr *)arg;
174 tn = <->loc;
175 tn->isr = Xisrclk;
176 coherence();
177
178 timerintr(ureg, 0);
179
180 #ifdef watchdog_not_bloody_useless
181 /* appease the dogs */
182 wd = <->wd;
183 if (wd->cnt == 0 &&
184 (wd->ctl & (Wdog | Wdogena | Tintena)) == (Wdog | Wdogena))
185 panic("cpu%d: zero watchdog count but no system reset",
186 m->machno);
187 wd->load = Dogtimeout - 1;
188 coherence();
189 #endif
190 SET(wd); USED(wd);
191 tegclockintr();
192
193 mpclocksanity();
194 }
195
196 void
clockprod(Ureg * ureg)197 clockprod(Ureg *ureg)
198 {
199 Ltimer *tn;
200
201 timerintr(ureg, 0);
202 tegclockintr();
203 if (m->machno != 0) { /* cpu1 gets stuck */
204 tn = &((Ploctmr *)soc.loctmr)->loc;
205 setltimer(tn, Tcycles);
206 }
207 }
208
209 static void
clockreset(Ltimer * tn)210 clockreset(Ltimer *tn)
211 {
212 if (probeaddr((uintptr)tn) < 0)
213 panic("no clock at %#p", tn);
214 tn->ctl = 0;
215 coherence();
216 }
217
218 void
watchdogoff(Ltimer * wd)219 watchdogoff(Ltimer *wd)
220 {
221 wd->ctl &= ~Wdogena;
222 coherence();
223 wd->wddis = Wdoff1;
224 coherence();
225 wd->wddis = Wdoff2;
226 coherence();
227 }
228
229 /* clear any pending watchdog intrs or causes */
230 void
wdogclrintr(Ltimer * wd)231 wdogclrintr(Ltimer *wd)
232 {
233 #ifdef watchdog_not_bloody_useless
234 wd->isr = Xisrclk;
235 coherence();
236 wd->wdrst = Wdrst;
237 coherence();
238 #endif
239 USED(wd);
240 }
241
242 /*
243 * stop clock interrupts on this cpu and disable the local watchdog timer,
244 * and, if on cpu0, shutdown the shared tegra2 watchdog timer.
245 */
246 void
clockshutdown(void)247 clockshutdown(void)
248 {
249 Ploctmr *lt;
250
251 lt = (Ploctmr *)soc.loctmr;
252 clockreset(<->loc);
253 watchdogoff(<->wd);
254
255 tegclockshutdown();
256 }
257
258 enum {
259 Instrs = 10*Mhz,
260 };
261
262 /* we assume that perfticks are microseconds */
263 static long
issue1loop(void)264 issue1loop(void)
265 {
266 register int i;
267 long st;
268
269 i = Instrs;
270 st = perfticks();
271 do {
272 --i; --i; --i; --i; --i; --i; --i; --i; --i; --i;
273 --i; --i; --i; --i; --i; --i; --i; --i; --i; --i;
274 --i; --i; --i; --i; --i; --i; --i; --i; --i; --i;
275 --i; --i; --i; --i; --i; --i; --i; --i; --i; --i;
276 --i; --i; --i; --i; --i; --i; --i; --i; --i; --i;
277 --i; --i; --i; --i; --i; --i; --i; --i; --i; --i;
278 --i; --i; --i; --i; --i; --i; --i; --i; --i; --i;
279 --i; --i; --i; --i; --i; --i; --i; --i; --i; --i;
280 --i; --i; --i; --i; --i; --i; --i; --i; --i; --i;
281 --i; --i; --i; --i; --i; --i; --i; --i; --i;
282 } while(--i >= 0);
283 return perfticks() - st;
284 }
285
286 static long
issue2loop(void)287 issue2loop(void)
288 {
289 register int i, j;
290 long st;
291
292 i = Instrs / 2; /* j gets half the decrements */
293 j = 0;
294 st = perfticks();
295 do {
296 --j; --i; --j; --i; --j; --i; --j; --i; --j;
297 --i; --j; --i; --j; --i; --j; --i; --j; --i; --j;
298 --i; --j; --i; --j; --i; --j; --i; --j; --i; --j;
299 --i; --j; --i; --j; --i; --j; --i; --j; --i; --j;
300 --i; --j; --i; --j; --i; --j; --i; --j; --i; --j;
301 --i; --j; --i; --j; --i; --j; --i; --j; --i; --j;
302 --i; --j; --i; --j; --i; --j; --i; --j; --i; --j;
303 --i; --j; --i; --j; --i; --j; --i; --j; --i; --j;
304 --i; --j; --i; --j; --i; --j; --i; --j; --i; --j;
305 --i; --j; --i; --j; --i; --j; --i; --j; --i; --j;
306
307 --i; --j; --i; --j; --i; --j; --i; --j; --i; --j;
308 --i; --j; --i; --j; --i; --j; --i; --j; --i; --j;
309 --i; --j; --i; --j; --i; --j; --i; --j; --i; --j;
310 --i; --j; --i; --j; --i; --j; --i; --j; --i; --j;
311 --i; --j; --i; --j; --i; --j; --i; --j; --i; --j;
312 --i; --j; --i; --j; --i; --j; --i; --j; --i; --j;
313 --i; --j; --i; --j; --i; --j; --i; --j; --i; --j;
314 --i; --j; --i; --j; --i; --j; --i; --j; --i; --j;
315 --i; --j; --i; --j; --i; --j; --i; --j; --i; --j;
316 --i; --j; --i; --j; --i; --j; --i; --j; --i; --j;
317 } while(--i >= 0);
318 return perfticks() - st;
319 }
320
321 /* estimate instructions/s. */
322 static void
guessmips(long (* loop)(void),char * lab)323 guessmips(long (*loop)(void), char *lab)
324 {
325 int s;
326 long tcks;
327
328 do {
329 s = splhi();
330 tcks = loop();
331 splx(s);
332 if (tcks < 0)
333 iprint("again...");
334 } while (tcks < 0);
335 /*
336 * Instrs instructions took tcks ticks @ Basetickfreq Hz.
337 * round the result.
338 */
339 s = (((vlong)Basetickfreq * Instrs) / tcks + 500000) / 1000000;
340 if (Debug)
341 iprint("%ud mips (%s-issue)", s, lab);
342 USED(s);
343 }
344
345 void
wdogintr(Ureg *,void * ltmr)346 wdogintr(Ureg *, void *ltmr)
347 {
348 #ifdef watchdog_not_bloody_useless
349 Ltimer *wd;
350
351 wd = ltmr;
352 fired++;
353 wdogclrintr(wd);
354 #endif
355 USED(ltmr);
356 }
357
358 static void
ckcounting(Ltimer * lt)359 ckcounting(Ltimer *lt)
360 {
361 ulong old;
362
363 old = lt->cnt;
364 if (old == lt->cnt)
365 delay(1);
366 if (old == lt->cnt)
367 panic("cpu%d: watchdog timer not counting down", m->machno);
368 }
369
370 /* test fire with interrupt to see that it's working */
371 static void
ckwatchdog(Ltimer * wd)372 ckwatchdog(Ltimer *wd)
373 {
374 #ifdef watchdog_not_bloody_useless
375 int s;
376
377 fired = 0;
378 wd->load = Tcycles - 1;
379 coherence();
380 /* Tintena is supposed to be ignored in watchdog mode */
381 wd->ctl |= Wdogena | Tintena;
382 coherence();
383
384 ckcounting(wd);
385
386 s = spllo();
387 delay(2 * 1000/HZ);
388 splx(s);
389 if (!fired)
390 /* useless local watchdog */
391 iprint("cpu%d: local watchdog failed to interrupt\n", m->machno);
392 /* clean up */
393 wd->ctl &= ~Wdogena;
394 coherence();
395 #endif
396 USED(wd);
397 }
398
399 static void
startwatchdog(void)400 startwatchdog(void)
401 {
402 #ifdef watchdog_not_bloody_useless
403 Ltimer *wd;
404 Ploctmr *lt;
405
406 lt = (Ploctmr *)soc.loctmr;
407 wd = <->wd;
408 watchdogoff(wd);
409 wdogclrintr(wd);
410 irqenable(Wdtmrirq, wdogintr, wd, "watchdog");
411
412 ckwatchdog(wd);
413
414 /* set up for normal use, causing reset */
415 wd->ctl &= ~Tintena; /* reset, don't interrupt */
416 coherence();
417 wd->ctl |= Wdog;
418 coherence();
419 wd->load = Dogtimeout - 1;
420 coherence();
421 wd->ctl |= Wdogena;
422 coherence();
423
424 ckcounting(wd);
425 #endif
426 }
427
428 static void
clock0init(Ltimer * tn)429 clock0init(Ltimer *tn)
430 {
431 int s;
432 ulong old, fticks;
433
434 /*
435 * calibrate fastclock
436 */
437 s = splhi();
438 tn->load = ~0ul >> 1;
439 coherence();
440 tn->ctl = Tmrena;
441 coherence();
442
443 old = perfticks();
444 fticks = tn->cnt;
445 delay(1);
446 fticks = abs(tn->cnt - fticks);
447 old = perfticks() - old;
448 splx(s);
449 if (Debug)
450 iprint("cpu%d: fastclock %ld/%ldµs = %ld fastticks/µs (MHz)\n",
451 m->machno, fticks, old, (fticks + old/2 - 1) / old);
452 USED(fticks, old);
453
454 if (Debug)
455 iprint("cpu%d: ", m->machno);
456 guessmips(issue1loop, "single");
457 if (Debug)
458 iprint(", ");
459 guessmips(issue2loop, "dual");
460 if (Debug)
461 iprint("\n");
462
463 /*
464 * m->delayloop should be the number of delay loop iterations
465 * needed to consume 1 ms. 2 is instr'ns in the delay loop.
466 */
467 m->delayloop = m->cpuhz / (1000 * 2);
468 // iprint("cpu%d: m->delayloop = %lud\n", m->machno, m->delayloop);
469
470 tegclock0init();
471 }
472
473 /*
474 * the local timer is the interrupting timer and does not
475 * participate in measuring time. It is initially set to HZ.
476 */
477 void
clockinit(void)478 clockinit(void)
479 {
480 ulong old;
481 Ltimer *tn;
482 Ploctmr *lt;
483
484 clockshutdown();
485
486 /* turn my cycle counter on */
487 cpwrsc(0, CpCLD, CpCLDena, CpCLDenacyc, 1<<31);
488
489 /* turn all my counters on and clear my cycle counter */
490 cpwrsc(0, CpCLD, CpCLDena, CpCLDenapmnc, 1<<2 | 1);
491
492 /* let users read my cycle counter directly */
493 cpwrsc(0, CpCLD, CpCLDuser, CpCLDenapmnc, 1);
494
495 /* verify µs counter sanity */
496 tegclockinit();
497
498 lt = (Ploctmr *)soc.loctmr;
499 tn = <->loc;
500 if (m->machno == 0)
501 irqenable(Loctmrirq, clockintr, lt, "clock");
502 else
503 intcunmask(Loctmrirq);
504
505 /*
506 * verify sanity of local timer
507 */
508 tn->load = Clockfreqbase / 1000;
509 tn->isr = Xisrclk;
510 coherence();
511 tn->ctl = Tmrena;
512 coherence();
513
514 old = tn->cnt;
515 delay(5);
516 /* m->ticks won't be incremented here because timersinit hasn't run. */
517 if (tn->cnt == old)
518 panic("cpu%d: clock not ticking at all", m->machno);
519 else if ((long)tn->cnt > 0)
520 panic("cpu%d: clock ticking slowly", m->machno);
521
522 if (m->machno == 0)
523 clock0init(tn);
524
525 /* if pci gets stuck, maybe one of the many watchdogs will nuke us. */
526 startwatchdog();
527
528 /*
529 * desynchronize the processor clocks so that they all don't
530 * try to resched at the same time.
531 */
532 delay(m->machno*2);
533 setltimer(tn, Tcycles);
534 }
535
536 /* our fastticks are at 1MHz (Basetickfreq), so the conversion is trivial. */
537 ulong
s(void)538 µs(void)
539 {
540 return fastticks2us(fastticks(nil));
541 }
542
543 /* Tval is supposed to be in fastticks units. */
544 void
timerset(Tval next)545 timerset(Tval next)
546 {
547 int s;
548 long offset;
549 Ltimer *tn;
550
551 tn = &((Ploctmr *)soc.loctmr)->loc;
552 s = splhi();
553 offset = fastticks2us(next - fastticks(nil));
554 /* offset is now in µs (MHz); convert to Clockfreqbase Hz. */
555 offset *= Clockfreqbase / Mhz;
556 if(offset < MinPeriod)
557 offset = MinPeriod;
558 else if(offset > MaxPeriod)
559 offset = MaxPeriod;
560
561 setltimer(tn, offset);
562 splx(s);
563 }
564
565 static ulong
cpucycles(void)566 cpucycles(void) /* cpu clock rate, except when waiting for intr (unused) */
567 {
568 ulong v;
569
570 /* reads 32-bit cycle counter (counting up) */
571 // v = cprdsc(0, CpCLD, CpCLDcyc, 0);
572 v = getcyc(); /* fast asm */
573 /* keep it non-negative; prevent m->fastclock ever going to 0 */
574 return v == 0? 1: v;
575 }
576
577 long
lcycles(void)578 lcycles(void)
579 {
580 return perfticks();
581 }
582
583 uvlong
fastticks(uvlong * hz)584 fastticks(uvlong *hz)
585 {
586 int s;
587 ulong newticks;
588 Vlong *fcp;
589
590 if(hz)
591 *hz = Basetickfreq;
592
593 fcp = (Vlong *)&m->fastclock;
594 /* avoid reentry on interrupt or trap, to prevent recursion */
595 s = splhi();
596 newticks = perfticks();
597 if(newticks < fcp->low) /* low word must have wrapped */
598 fcp->high++;
599 fcp->low = newticks;
600 splx(s);
601
602 if (fcp->low == 0 && fcp->high == 0 && m->ticks > HZ/10)
603 panic("fastticks: zero m->fastclock; ticks %lud fastclock %#llux",
604 m->ticks, m->fastclock);
605 return m->fastclock;
606 }
607
608 void
microdelay(int l)609 microdelay(int l)
610 {
611 for (l = l * (vlong)m->delayloop / 1000; --l >= 0; )
612 ;
613 }
614
615 void
delay(int l)616 delay(int l)
617 {
618 int i, d;
619
620 d = m->delayloop;
621 while(--l >= 0)
622 for (i = d; --i >= 0; )
623 ;
624 }
625