1 /* $OpenBSD: loongson3_intr.c,v 1.8 2022/08/22 00:35:07 cheloha Exp $ */
2
3 /*
4 * Copyright (c) 2016 Visa Hankala
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19 /*
20 * Loongson 3A interrupt handling.
21 */
22
23 #include <sys/param.h>
24 #include <sys/systm.h>
25 #include <sys/atomic.h>
26 #include <sys/malloc.h>
27
28 #include <machine/autoconf.h>
29 #include <machine/intr.h>
30 #include <machine/loongson3.h>
31
32 #include <mips64/mips_cpu.h>
33
34 uint32_t loongson3_ht_intr(uint32_t, struct trapframe *);
35 uint32_t loongson3_intr(uint32_t, struct trapframe *);
36 void loongson3_splx(int);
37
38 const struct pic *loongson3_ht_pic;
39 paddr_t loongson3_ht_cfg_base;
40
41 #define HT_REGVAL(offset) REGVAL32(loongson3_ht_cfg_base + (offset))
42
43 /*
44 * Interrupt handlers are sorted by interrupt priority level.
45 */
46
47 struct intrhand *loongson3_ht_intrhand[LS3_HT_IRQ_NUM];
48 uint32_t loongson3_ht_imask[NIPLS];
49 uint32_t loongson3_ht_intem;
50
51 struct intrhand *loongson3_intrhand[LS3_IRQ_NUM];
52 uint32_t loongson3_imask[NIPLS];
53 uint32_t loongson3_intem;
54
55 static inline int
next_irq(uint32_t * isr)56 next_irq(uint32_t *isr)
57 {
58 uint64_t tmp = *isr;
59 uint32_t irq;
60
61 if (tmp == 0)
62 return -1;
63
64 asm volatile (
65 " .set push\n"
66 " .set mips64\n"
67 " dclz %0, %0\n"
68 " .set pop\n"
69 : "=r" (tmp) : "0" (tmp));
70
71 irq = 63u - tmp;
72 *isr &= ~(1u << irq);
73 return irq;
74 }
75
76 void
loongson3_intr_init(void)77 loongson3_intr_init(void)
78 {
79 uint32_t boot_core = LS3_COREID(loongson3_get_cpuid());
80 int ht_node = 0;
81 int core, node;
82 int i, ipl, irq;
83
84 if (loongson_ver == 0x3b)
85 ht_node = 1;
86 loongson3_ht_cfg_base = LS3_HT1_CFG_BASE(ht_node);
87
88 for (node = 0; node < nnodes; node++) {
89 /* Disable all interrupts. */
90 REGVAL(LS3_IRT_INTENCLR(node)) = ~0u;
91
92 /* Disable all HT interrupts and ack any pending ones. */
93 for (i = 0; i < 8; i++) {
94 HT_REGVAL(LS3_HT_IMR_OFFSET(node)) = 0;
95 HT_REGVAL(LS3_HT_ISR_OFFSET(node)) = ~0u;
96 }
97
98 /* Disable IPIs on every core. */
99 for (core = 0; core < 4; core++)
100 REGVAL32(LS3_IPI_BASE(node, core) + LS3_IPI_IMR) = 0u;
101 }
102
103 /*
104 * On the boot node, route all interrupts to the boot core.
105 * Assign a separate priority level to HT interrupts to process them
106 * in a dedicated handler.
107 */
108 for (irq = 0; irq < LS3_IRQ_NUM; irq++) {
109 if (LS3_IRQ_IS_HT(irq))
110 ipl = 0;
111 else
112 ipl = 1;
113 REGVAL8(LS3_IRT_ENTRY(0, irq)) = LS3_IRT_ROUTE(boot_core, ipl);
114 }
115
116 /* Enable HT interrupt vectors 0-63 on the boot node's router. */
117 loongson3_intem |= 1u << LS3_IRQ_HT1(0);
118
119 register_splx_handler(loongson3_splx);
120 set_intr(INTPRI_CLOCK + 1, CR_INT_1, loongson3_intr);
121 set_intr(INTPRI_CLOCK + 2, CR_INT_0, loongson3_ht_intr);
122 }
123
124 void
loongson3_prop_imask(uint32_t * imask)125 loongson3_prop_imask(uint32_t *imask)
126 {
127 imask[IPL_NONE] = 0;
128 imask[IPL_NET] |= imask[IPL_BIO];
129 imask[IPL_TTY] |= imask[IPL_NET];
130 imask[IPL_VM] |= imask[IPL_TTY];
131 imask[IPL_CLOCK] |= imask[IPL_VM];
132 imask[IPL_HIGH] |= imask[IPL_CLOCK];
133 imask[IPL_IPI] |= imask[IPL_HIGH];
134 }
135
136 void
loongson3_update_imask(void)137 loongson3_update_imask(void)
138 {
139 uint32_t ipls[LS3_IRQ_NUM];
140 uint32_t ipls_ht[LS3_HT_IRQ_NUM];
141 struct intrhand *ih;
142 register_t sr;
143 int irq, level;
144
145 sr = disableintr();
146
147 for (irq = 0; irq < LS3_IRQ_NUM; irq++) {
148 ipls[irq] = 0;
149 for (ih = loongson3_intrhand[irq]; ih != NULL; ih = ih->ih_next)
150 ipls[irq] |= 1u << ih->ih_level;
151 }
152 for (irq = 0; irq < LS3_HT_IRQ_NUM; irq++) {
153 ipls_ht[irq] = 0;
154 for (ih = loongson3_ht_intrhand[irq]; ih != NULL;
155 ih = ih->ih_next) {
156 ipls_ht[irq] |= 1u << ih->ih_level;
157 ipls[LS3_IRQ_HT1(0)] |= 1u << ih->ih_level;
158 }
159 }
160
161 for (level = IPL_NONE; level < NIPLS; level++) {
162 loongson3_imask[level] = 0;
163 for (irq = 0; irq < LS3_IRQ_NUM; irq++) {
164 if (ipls[irq] & (1u << level))
165 loongson3_imask[level] |= 1u << irq;
166 }
167
168 loongson3_ht_imask[level] = 0;
169 for (irq = 0; irq < LS3_HT_IRQ_NUM; irq++) {
170 if (ipls_ht[irq] & (1u << level))
171 loongson3_ht_imask[level] |= 1u << irq;
172 }
173 }
174
175 loongson3_prop_imask(loongson3_imask);
176 loongson3_prop_imask(loongson3_ht_imask);
177
178 setsr(sr);
179 }
180
181 void
loongson3_intr_insert(struct intrhand ** list,struct intrhand * ih,int level)182 loongson3_intr_insert(struct intrhand **list, struct intrhand *ih, int level)
183 {
184 struct intrhand *next, *prev;
185
186 if (*list != NULL) {
187 for (prev = NULL, next = *list;
188 next != NULL && next->ih_level >= level;
189 prev = next, next = next->ih_next)
190 continue;
191 if (prev != NULL)
192 prev->ih_next = ih;
193 else
194 *list = ih;
195 ih->ih_next = next;
196 } else
197 *list = ih;
198 }
199
200 void
loongson3_intr_remove(struct intrhand ** list,struct intrhand * ih)201 loongson3_intr_remove(struct intrhand **list, struct intrhand *ih)
202 {
203 struct intrhand *prev;
204
205 if (*list == ih) {
206 *list = (*list)->ih_next;
207 } else {
208 for (prev = *list; prev != NULL; prev = prev->ih_next) {
209 if (prev->ih_next == ih) {
210 prev->ih_next = ih->ih_next;
211 break;
212 }
213 }
214 if (prev == NULL)
215 panic("%s: intrhand %p has not been registered",
216 __func__, ih);
217 }
218 }
219
220 void *
loongson3_intr_establish(int irq,int level,int (* func)(void *),void * arg,const char * name)221 loongson3_intr_establish(int irq, int level, int (*func)(void *), void *arg,
222 const char *name)
223 {
224 struct intrhand *ih;
225 register_t sr;
226 int flags, s;
227
228 if ((unsigned int)irq >= LS3_IRQ_NUM || LS3_IRQ_IS_HT(irq))
229 return NULL;
230
231 flags = (level & IPL_MPSAFE) ? IH_MPSAFE : 0;
232 level &= ~IPL_MPSAFE;
233
234 ih = malloc(sizeof(*ih), M_DEVBUF, M_NOWAIT);
235 if (ih == NULL)
236 return NULL;
237
238 ih->ih_next = NULL;
239 ih->ih_fun = func;
240 ih->ih_arg = arg;
241 ih->ih_level = level;
242 ih->ih_irq = irq;
243 ih->ih_flags = flags;
244 evcount_attach(&ih->ih_count, name, (void *)&ih->ih_irq);
245
246 sr = disableintr();
247 s = splhigh();
248
249 loongson3_intr_insert(&loongson3_intrhand[irq], ih, level);
250
251 loongson3_intem |= 1u << irq;
252 loongson3_update_imask();
253
254 HT_REGVAL(LS3_HT_IMR_OFFSET(0)) = loongson3_ht_intem;
255
256 splx(s);
257 setsr(sr);
258
259 return ih;
260 }
261
262 void
loongson3_intr_disestablish(void * ihp)263 loongson3_intr_disestablish(void *ihp)
264 {
265 struct intrhand *ih = ihp;
266 register_t sr;
267 int s;
268
269 sr = disableintr();
270 s = splhigh();
271
272 loongson3_intr_remove(&loongson3_intrhand[ih->ih_irq], ih);
273 free(ih, M_DEVBUF, sizeof(*ih));
274
275 loongson3_update_imask();
276 splx(s);
277 setsr(sr);
278 }
279
280 void *
loongson3_ht_intr_establish(int irq,int level,int (* func)(void *),void * arg,const char * name)281 loongson3_ht_intr_establish(int irq, int level, int (*func)(void *), void *arg,
282 const char *name)
283 {
284 struct intrhand *ih;
285 register_t sr;
286 int flags, s;
287
288 if ((unsigned int)irq >= LS3_HT_IRQ_NUM)
289 return NULL;
290
291 flags = (level & IPL_MPSAFE) ? IH_MPSAFE : 0;
292 level &= ~IPL_MPSAFE;
293
294 ih = malloc(sizeof(*ih), M_DEVBUF, M_NOWAIT);
295 if (ih == NULL)
296 return NULL;
297
298 ih->ih_next = NULL;
299 ih->ih_fun = func;
300 ih->ih_arg = arg;
301 ih->ih_level = level;
302 ih->ih_irq = irq;
303 ih->ih_flags = flags;
304 evcount_attach(&ih->ih_count, name, (void *)&ih->ih_irq);
305
306 sr = disableintr();
307 s = splhigh();
308
309 loongson3_intr_insert(&loongson3_ht_intrhand[irq], ih, level);
310
311 loongson3_ht_intem |= 1u << irq;
312 loongson3_update_imask();
313
314 loongson3_ht_pic->pic_unmask(irq);
315
316 splx(s);
317 setsr(sr);
318
319 return ih;
320 }
321
322 void
loongson3_ht_intr_disestablish(void * ihp)323 loongson3_ht_intr_disestablish(void *ihp)
324 {
325 struct intrhand *ih = ihp;
326 register_t sr;
327 int irq = ih->ih_irq;
328 int s;
329
330 sr = disableintr();
331 s = splhigh();
332
333 loongson3_intr_remove(&loongson3_ht_intrhand[irq], ih);
334 free(ih, M_DEVBUF, sizeof(*ih));
335
336 if (loongson3_ht_intrhand[irq] == NULL) {
337 loongson3_ht_pic->pic_mask(irq);
338 loongson3_ht_intem &= ~(1u << irq);
339 }
340
341 loongson3_update_imask();
342
343 splx(s);
344 setsr(sr);
345 }
346
347 void
loongson3_splx(int newipl)348 loongson3_splx(int newipl)
349 {
350 struct cpu_info *ci = curcpu();
351
352 ci->ci_ipl = newipl;
353
354 if (CPU_IS_PRIMARY(ci))
355 REGVAL(LS3_IRT_INTENSET(0)) =
356 loongson3_intem & ~loongson3_imask[newipl];
357
358 /* Trigger deferred clock interrupt if it is now unmasked. */
359 if (ci->ci_clock_deferred && newipl < IPL_CLOCK)
360 md_triggerclock();
361
362 if (ci->ci_softpending != 0 && newipl < IPL_SOFTINT)
363 setsoftintr0();
364 }
365
366 uint32_t
loongson3_intr(uint32_t pending,struct trapframe * frame)367 loongson3_intr(uint32_t pending, struct trapframe *frame)
368 {
369 struct cpu_info *ci = curcpu();
370 struct intrhand *ih;
371 uint32_t imr, isr, mask;
372 int handled;
373 int ipl, irq;
374 #ifdef MULTIPROCESSOR
375 register_t sr;
376 int need_lock;
377 #endif
378
379 isr = REGVAL(LS3_IRT_INTISR(0));
380 imr = REGVAL(LS3_IRT_INTEN(0)) & ~LS3_IRQ_HT_MASK;
381
382 isr &= imr;
383 if (isr == 0)
384 return 0;
385
386 /* Mask pending interrupts. */
387 REGVAL(LS3_IRT_INTENCLR(0)) = isr;
388
389 if ((mask = isr & loongson3_imask[frame->ipl]) != 0) {
390 isr &= ~mask;
391 imr &= ~mask;
392 }
393 if (isr == 0)
394 return pending;
395
396 ipl = ci->ci_ipl;
397
398 while ((irq = next_irq(&isr)) >= 0) {
399 handled = 0;
400 for (ih = loongson3_intrhand[irq]; ih != NULL;
401 ih = ih->ih_next) {
402 splraise(ih->ih_level);
403 #ifdef MULTIPROCESSOR
404 if (ih->ih_level < IPL_IPI) {
405 sr = getsr();
406 ENABLEIPI();
407 }
408 if (ih->ih_flags & IH_MPSAFE)
409 need_lock = 0;
410 else
411 need_lock = 1;
412 if (need_lock)
413 __mp_lock(&kernel_lock);
414 #endif
415 if (ih->ih_fun(ih->ih_arg) != 0) {
416 atomic_inc_long((unsigned long *)
417 &ih->ih_count.ec_count);
418 handled = 1;
419 }
420 #ifdef MULTIPROCESSOR
421 if (need_lock)
422 __mp_unlock(&kernel_lock);
423 if (ih->ih_level < IPL_IPI)
424 setsr(sr);
425 #endif
426 }
427 if (!handled)
428 printf("spurious interrupt %d\n", irq);
429 }
430
431 ci->ci_ipl = ipl;
432
433 /* Re-enable processed interrupts. */
434 REGVAL(LS3_IRT_INTENSET(0)) = imr;
435
436 return pending;
437 }
438
439 uint32_t
loongson3_ht_intr(uint32_t pending,struct trapframe * frame)440 loongson3_ht_intr(uint32_t pending, struct trapframe *frame)
441 {
442 struct cpu_info *ci = curcpu();
443 struct intrhand *ih;
444 uint32_t imr, isr, mask;
445 int handled;
446 int ipl, irq;
447 #ifdef MULTIPROCESSOR
448 register_t sr;
449 int need_lock;
450 #endif
451
452 isr = HT_REGVAL(LS3_HT_ISR_OFFSET(0));
453 imr = HT_REGVAL(LS3_HT_IMR_OFFSET(0));
454
455 isr &= imr;
456 if (isr == 0)
457 return 0;
458
459 /* Mask pending HT interrupts. */
460 REGVAL(LS3_IRT_INTENCLR(0)) = 1u << LS3_IRQ_HT1(0);
461
462 if ((mask = isr & loongson3_ht_imask[frame->ipl]) != 0) {
463 isr &= ~mask;
464 imr &= ~mask;
465 }
466 if (isr == 0)
467 return pending;
468
469 /* Acknowledge HT interrupts that will be processed. */
470 HT_REGVAL(LS3_HT_ISR_OFFSET(0)) = isr;
471
472 ipl = ci->ci_ipl;
473
474 while ((irq = next_irq(&isr)) >= 0) {
475 handled = 0;
476 for (ih = loongson3_ht_intrhand[irq]; ih != NULL;
477 ih = ih->ih_next) {
478 splraise(ih->ih_level);
479 #ifdef MULTIPROCESSOR
480 if (ih->ih_level < IPL_IPI) {
481 sr = getsr();
482 ENABLEIPI();
483 }
484 if (ih->ih_flags & IH_MPSAFE)
485 need_lock = 0;
486 else
487 need_lock = 1;
488 if (need_lock)
489 __mp_lock(&kernel_lock);
490 #endif
491 if (ih->ih_fun(ih->ih_arg) != 0) {
492 atomic_inc_long((unsigned long *)
493 &ih->ih_count.ec_count);
494 handled = 1;
495 }
496 #ifdef MULTIPROCESSOR
497 if (need_lock)
498 __mp_unlock(&kernel_lock);
499 if (ih->ih_level < IPL_IPI)
500 setsr(sr);
501 #endif
502 }
503 #ifdef notyet
504 if (!handled)
505 printf("spurious HT interrupt %d\n", irq);
506 #endif
507
508 loongson3_ht_pic->pic_eoi(irq);
509 }
510
511 ci->ci_ipl = ipl;
512
513 /* Re-enable HT interrupts. */
514 REGVAL(LS3_IRT_INTENSET(0)) = 1u << LS3_IRQ_HT1(0);
515
516 return pending;
517 }
518
519 void
loongson3_register_ht_pic(const struct pic * pic)520 loongson3_register_ht_pic(const struct pic *pic)
521 {
522 loongson3_ht_pic = pic;
523 }
524