xref: /netbsd-src/sys/arch/arm/pic/pic.c (revision c2f76ff004a2cb67efe5b12d97bd3ef7fe89e18d) 
1 /*	$NetBSD: pic.c,v 1.6 2010/08/31 14:33:41 kiyohara Exp $	*/
2 /*-
3  * Copyright (c) 2008 The NetBSD Foundation, Inc.
4  * All rights reserved.
5  *
6  * This code is derived from software contributed to The NetBSD Foundation
7  * by Matt Thomas.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
19  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
20  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
22  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28  * POSSIBILITY OF SUCH DAMAGE.
29  */
30 #include <sys/cdefs.h>
31 __KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.6 2010/08/31 14:33:41 kiyohara Exp $");
32 
33 #define _INTR_PRIVATE
34 #include <sys/param.h>
35 #include <sys/evcnt.h>
36 #include <sys/atomic.h>
37 #include <sys/malloc.h>
38 #include <sys/mallocvar.h>
39 #include <sys/atomic.h>
40 
41 #include <arm/armreg.h>
42 #include <arm/cpu.h>
43 #include <arm/cpufunc.h>
44 
45 #include <arm/pic/picvar.h>
46 
47 MALLOC_DEFINE(M_INTRSOURCE, "intrsource", "interrupt source");
48 
49 static uint32_t
50 	pic_find_pending_irqs_by_ipl(struct pic_softc *, size_t, uint32_t, int);
51 static struct pic_softc *
52 	pic_list_find_pic_by_pending_ipl(uint32_t);
53 static void
54 	pic_deliver_irqs(struct pic_softc *, int, void *);
55 static void
56 	pic_list_deliver_irqs(register_t, int, void *);
57 
58 struct pic_softc *pic_list[PIC_MAXPICS];
59 #if PIC_MAXPICS > 32
60 #error PIC_MAXPICS > 32 not supported
61 #endif
62 volatile uint32_t pic_blocked_pics;
63 volatile uint32_t pic_pending_pics;
64 volatile uint32_t pic_pending_ipls;
65 struct intrsource *pic_sources[PIC_MAXMAXSOURCES];
66 struct intrsource *pic__iplsources[PIC_MAXMAXSOURCES];
67 struct intrsource **pic_iplsource[NIPL] = {
68 	[0 ... NIPL-1] = pic__iplsources,
69 };
70 size_t pic_ipl_offset[NIPL+1];
71 size_t pic_sourcebase;
72 static struct evcnt pic_deferral_ev =
73     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "deferred", "intr");
74 EVCNT_ATTACH_STATIC(pic_deferral_ev);
75 
76 
77 
78 int
79 pic_handle_intr(void *arg)
80 {
81 	struct pic_softc * const pic = arg;
82 	int rv;
83 
84 	rv = (*pic->pic_ops->pic_find_pending_irqs)(pic);
85 
86 	return rv > 0;
87 }
88 
89 void
90 pic_mark_pending_source(struct pic_softc *pic, struct intrsource *is)
91 {
92 	const uint32_t ipl_mask = __BIT(is->is_ipl);
93 
94 	atomic_or_32(&pic->pic_pending_irqs[is->is_irq >> 5],
95 	    __BIT(is->is_irq & 0x1f));
96 
97 	atomic_or_32(&pic->pic_pending_ipls, ipl_mask);
98 	atomic_or_32(&pic_pending_ipls, ipl_mask);
99 	atomic_or_32(&pic_pending_pics, __BIT(pic->pic_id));
100 }
101 
102 void
103 pic_mark_pending(struct pic_softc *pic, int irq)
104 {
105 	struct intrsource * const is = pic->pic_sources[irq];
106 
107 	KASSERT(irq < pic->pic_maxsources);
108 	KASSERT(is != NULL);
109 
110 	pic_mark_pending_source(pic, is);
111 }
112 
113 uint32_t
114 pic_mark_pending_sources(struct pic_softc *pic, size_t irq_base,
115 	uint32_t pending)
116 {
117 	struct intrsource ** const isbase = &pic->pic_sources[irq_base];
118 	struct intrsource *is;
119 	volatile uint32_t *ipending = &pic->pic_pending_irqs[irq_base >> 5];
120 	uint32_t ipl_mask = 0;
121 
122 	if (pending == 0)
123 		return ipl_mask;
124 
125 	KASSERT((irq_base & 31) == 0);
126 
127 	(*pic->pic_ops->pic_block_irqs)(pic, irq_base, pending);
128 
129 	atomic_or_32(ipending, pending);
130         while (pending != 0) {
131 		int n = ffs(pending);
132 		if (n-- == 0)
133 			break;
134 		is = isbase[n];
135 		KASSERT(is != NULL);
136 		KASSERT(irq_base <= is->is_irq && is->is_irq < irq_base + 32);
137 		pending &= ~__BIT(n);
138 		ipl_mask |= __BIT(is->is_ipl);
139 	}
140 
141 	atomic_or_32(&pic->pic_pending_ipls, ipl_mask);
142 	atomic_or_32(&pic_pending_ipls, ipl_mask);
143 	atomic_or_32(&pic_pending_pics, __BIT(pic->pic_id));
144 
145 	return ipl_mask;
146 }
147 
148 uint32_t
149 pic_find_pending_irqs_by_ipl(struct pic_softc *pic, size_t irq_base,
150 	uint32_t pending, int ipl)
151 {
152 	uint32_t ipl_irq_mask = 0;
153 	uint32_t irq_mask;
154 
155 	for (;;) {
156 		int irq = ffs(pending);
157 		if (irq-- == 0)
158 			return ipl_irq_mask;
159 
160 		irq_mask = __BIT(irq);
161 		KASSERT(pic->pic_sources[irq_base + irq] != NULL);
162 		if (pic->pic_sources[irq_base + irq]->is_ipl == ipl)
163 			ipl_irq_mask |= irq_mask;
164 
165 		pending &= ~irq_mask;
166 	}
167 }
168 
169 void
170 pic_dispatch(struct intrsource *is, void *frame)
171 {
172 	int rv;
173 
174 	if (__predict_false(is->is_arg == NULL)
175 	    && __predict_true(frame != NULL)) {
176 		rv = (*is->is_func)(frame);
177 	} else if (__predict_true(is->is_arg != NULL)) {
178 		rv = (*is->is_func)(is->is_arg);
179 	} else {
180 		pic_deferral_ev.ev_count++;
181 		return;
182 	}
183 	is->is_ev.ev_count++;
184 }
185 
186 void
187 pic_deliver_irqs(struct pic_softc *pic, int ipl, void *frame)
188 {
189 	const uint32_t ipl_mask = __BIT(ipl);
190 	struct intrsource *is;
191 	volatile uint32_t *ipending = pic->pic_pending_irqs;
192 	volatile uint32_t *iblocked = pic->pic_blocked_irqs;
193 	size_t irq_base;
194 #if PIC_MAXSOURCES > 32
195 	size_t irq_count;
196 	int poi = 0;		/* Possibility of interrupting */
197 #endif
198 	uint32_t pending_irqs;
199 	uint32_t blocked_irqs;
200 	int irq;
201 	bool progress = false;
202 
203 	KASSERT(pic->pic_pending_ipls & ipl_mask);
204 
205 	irq_base = 0;
206 #if PIC_MAXSOURCES > 32
207 	irq_count = 0;
208 #endif
209 
210 	for (;;) {
211 		pending_irqs = pic_find_pending_irqs_by_ipl(pic, irq_base,
212 		    *ipending, ipl);
213 		KASSERT((pending_irqs & *ipending) == pending_irqs);
214 		KASSERT((pending_irqs & ~(*ipending)) == 0);
215 		if (pending_irqs == 0) {
216 #if PIC_MAXSOURCES > 32
217 			irq_count += 32;
218 			if (__predict_true(irq_count >= pic->pic_maxsources)) {
219 				if (!poi)
220 					/*Interrupt at this level was handled.*/
221 					break;
222 				irq_base = 0;
223 				irq_count = 0;
224 				poi = 0;
225 				ipending = pic->pic_pending_irqs;
226 				iblocked = pic->pic_blocked_irqs;
227 			} else {
228 				irq_base += 32;
229 				ipending++;
230 				iblocked++;
231 				KASSERT(irq_base <= pic->pic_maxsources);
232 			}
233 			continue;
234 #else
235 			break;
236 #endif
237 		}
238 		progress = true;
239 		blocked_irqs = 0;
240 		do {
241 			irq = ffs(pending_irqs) - 1;
242 			KASSERT(irq >= 0);
243 
244 			atomic_and_32(ipending, ~__BIT(irq));
245 			is = pic->pic_sources[irq_base + irq];
246 			if (is != NULL) {
247 				cpsie(I32_bit);
248 				pic_dispatch(is, frame);
249 				cpsid(I32_bit);
250 #if PIC_MAXSOURCES > 32
251 				/*
252 				 * There is a possibility of interrupting
253 				 * from cpsie() to cpsid().
254 				 */
255 				poi = 1;
256 #endif
257 				blocked_irqs |= __BIT(irq);
258 			} else {
259 				KASSERT(0);
260 			}
261 			pending_irqs = pic_find_pending_irqs_by_ipl(pic,
262 			    irq_base, *ipending, ipl);
263 		} while (pending_irqs);
264 		if (blocked_irqs) {
265 			atomic_or_32(iblocked, blocked_irqs);
266 			atomic_or_32(&pic_blocked_pics, __BIT(pic->pic_id));
267 		}
268 	}
269 
270 	KASSERT(progress);
271 	/*
272 	 * Since interrupts are disabled, we don't have to be too careful
273 	 * about these.
274 	 */
275 	if (atomic_and_32_nv(&pic->pic_pending_ipls, ~ipl_mask) == 0)
276 		atomic_and_32(&pic_pending_pics, ~__BIT(pic->pic_id));
277 }
278 
279 static void
280 pic_list_unblock_irqs(void)
281 {
282 	uint32_t blocked_pics = pic_blocked_pics;
283 
284 	pic_blocked_pics = 0;
285 	for (;;) {
286 		struct pic_softc *pic;
287 #if PIC_MAXSOURCES > 32
288 		volatile uint32_t *iblocked;
289 		uint32_t blocked;
290 		size_t irq_base;
291 #endif
292 
293 		int pic_id = ffs(blocked_pics);
294 		if (pic_id-- == 0)
295 			return;
296 
297 		pic = pic_list[pic_id];
298 		KASSERT(pic != NULL);
299 #if PIC_MAXSOURCES > 32
300 		for (irq_base = 0, iblocked = pic->pic_blocked_irqs;
301 		     irq_base < pic->pic_maxsources;
302 		     irq_base += 32, iblocked++) {
303 			if ((blocked = *iblocked) != 0) {
304 				(*pic->pic_ops->pic_unblock_irqs)(pic,
305 				    irq_base, blocked);
306 				atomic_and_32(iblocked, ~blocked);
307 			}
308 		}
309 #else
310 		KASSERT(pic->pic_blocked_irqs[0] != 0);
311 		(*pic->pic_ops->pic_unblock_irqs)(pic,
312 		    0, pic->pic_blocked_irqs[0]);
313 		pic->pic_blocked_irqs[0] = 0;
314 #endif
315 		blocked_pics &= ~__BIT(pic_id);
316 	}
317 }
318 
319 
320 struct pic_softc *
321 pic_list_find_pic_by_pending_ipl(uint32_t ipl_mask)
322 {
323 	uint32_t pending_pics = pic_pending_pics;
324 	struct pic_softc *pic;
325 
326 	for (;;) {
327 		int pic_id = ffs(pending_pics);
328 		if (pic_id-- == 0)
329 			return NULL;
330 
331 		pic = pic_list[pic_id];
332 		KASSERT(pic != NULL);
333 		if (pic->pic_pending_ipls & ipl_mask)
334 			return pic;
335 		pending_pics &= ~__BIT(pic_id);
336 	}
337 }
338 
339 void
340 pic_list_deliver_irqs(register_t psw, int ipl, void *frame)
341 {
342 	const uint32_t ipl_mask = __BIT(ipl);
343 	struct pic_softc *pic;
344 
345 	while ((pic = pic_list_find_pic_by_pending_ipl(ipl_mask)) != NULL) {
346 		pic_deliver_irqs(pic, ipl, frame);
347 		KASSERT((pic->pic_pending_ipls & ipl_mask) == 0);
348 	}
349 	atomic_and_32(&pic_pending_ipls, ~ipl_mask);
350 }
351 
352 void
353 pic_do_pending_ints(register_t psw, int newipl, void *frame)
354 {
355 	struct cpu_info * const ci = curcpu();
356 	if (__predict_false(newipl == IPL_HIGH))
357 		return;
358 	while ((pic_pending_ipls & ~__BIT(newipl)) > __BIT(newipl)) {
359 		KASSERT(pic_pending_ipls < __BIT(NIPL));
360 		for (;;) {
361 			int ipl = 31 - __builtin_clz(pic_pending_ipls);
362 			KASSERT(ipl < NIPL);
363 			if (ipl <= newipl)
364 				break;
365 
366 			ci->ci_cpl = ipl;
367 			pic_list_deliver_irqs(psw, ipl, frame);
368 			pic_list_unblock_irqs();
369 		}
370 	}
371 	if (ci->ci_cpl != newipl)
372 		ci->ci_cpl = newipl;
373 #ifdef __HAVE_FAST_SOFTINTS
374 	cpu_dosoftints();
375 #endif
376 }
377 
378 void
379 pic_add(struct pic_softc *pic, int irqbase)
380 {
381 	int slot, maybe_slot = -1;
382 
383 	for (slot = 0; slot < PIC_MAXPICS; slot++) {
384 		struct pic_softc * const xpic = pic_list[slot];
385 		if (xpic == NULL) {
386 			if (maybe_slot < 0)
387 				maybe_slot = slot;
388 			if (irqbase < 0)
389 				break;
390 			continue;
391 		}
392 		if (irqbase < 0 || xpic->pic_irqbase < 0)
393 			continue;
394 		if (irqbase >= xpic->pic_irqbase + xpic->pic_maxsources)
395 			continue;
396 		if (irqbase + pic->pic_maxsources <= xpic->pic_irqbase)
397 			continue;
398 		panic("pic_add: pic %s (%zu sources @ irq %u) conflicts"
399 		    " with pic %s (%zu sources @ irq %u)",
400 		    pic->pic_name, pic->pic_maxsources, irqbase,
401 		    xpic->pic_name, xpic->pic_maxsources, xpic->pic_irqbase);
402 	}
403 	slot = maybe_slot;
404 #if 0
405 	printf("%s: pic_sourcebase=%zu pic_maxsources=%zu\n",
406 	    pic->pic_name, pic_sourcebase, pic->pic_maxsources);
407 #endif
408 	KASSERT(pic_sourcebase + pic->pic_maxsources <= PIC_MAXMAXSOURCES);
409 
410 	pic->pic_sources = &pic_sources[pic_sourcebase];
411 	pic->pic_irqbase = irqbase;
412 	pic_sourcebase += pic->pic_maxsources;
413 	pic->pic_id = slot;
414 	pic_list[slot] = pic;
415 }
416 
417 int
418 pic_alloc_irq(struct pic_softc *pic)
419 {
420 	int irq;
421 
422 	for (irq = 0; irq < pic->pic_maxsources; irq++) {
423 		if (pic->pic_sources[irq] == NULL)
424 			return irq;
425 	}
426 
427 	return -1;
428 }
429 
430 void *
431 pic_establish_intr(struct pic_softc *pic, int irq, int ipl, int type,
432 	int (*func)(void *), void *arg)
433 {
434 	struct intrsource *is;
435 	int off, nipl;
436 
437 	if (pic->pic_sources[irq]) {
438 		printf("pic_establish_intr: pic %s irq %d already present\n",
439 		    pic->pic_name, irq);
440 		return NULL;
441 	}
442 
443 	is = malloc(sizeof(*is), M_INTRSOURCE, M_NOWAIT|M_ZERO);
444 	if (is == NULL)
445 		return NULL;
446 
447 	is->is_pic = pic;
448 	is->is_irq = irq;
449 	is->is_ipl = ipl;
450 	is->is_type = type;
451 	is->is_func = func;
452 	is->is_arg = arg;
453 
454 	if (pic->pic_ops->pic_source_name)
455 		(*pic->pic_ops->pic_source_name)(pic, irq, is->is_source,
456 		    sizeof(is->is_source));
457 	else
458 		snprintf(is->is_source, sizeof(is->is_source), "irq %d", irq);
459 
460 	evcnt_attach_dynamic(&is->is_ev, EVCNT_TYPE_INTR, NULL,
461 	    pic->pic_name, is->is_source);
462 
463 	pic->pic_sources[irq] = is;
464 
465 	/*
466 	 * First try to use an existing slot which is empty.
467 	 */
468 	for (off = pic_ipl_offset[ipl]; off < pic_ipl_offset[ipl+1]; off++) {
469 		if (pic__iplsources[off] == NULL) {
470 			is->is_iplidx = off - pic_ipl_offset[ipl];
471 			pic__iplsources[off] = is;
472 			return is;
473 		}
474 	}
475 
476 	/*
477 	 * Move up all the sources by one.
478  	 */
479 	if (ipl < NIPL) {
480 		off = pic_ipl_offset[ipl+1];
481 		memmove(&pic__iplsources[off+1], &pic__iplsources[off],
482 		    sizeof(pic__iplsources[0]) * (pic_ipl_offset[NIPL] - off));
483 	}
484 
485 	/*
486 	 * Advance the offset of all IPLs higher than this.  Include an
487 	 * extra one as well.  Thus the number of sources per ipl is
488 	 * pic_ipl_offset[ipl+1] - pic_ipl_offset[ipl].
489 	 */
490 	for (nipl = ipl + 1; nipl <= NIPL; nipl++)
491 		pic_ipl_offset[nipl]++;
492 
493 	/*
494 	 * Insert into the previously made position at the end of this IPL's
495 	 * sources.
496 	 */
497 	off = pic_ipl_offset[ipl + 1] - 1;
498 	is->is_iplidx = off - pic_ipl_offset[ipl];
499 	pic__iplsources[off] = is;
500 
501 	(*pic->pic_ops->pic_establish_irq)(pic, is);
502 
503 	(*pic->pic_ops->pic_unblock_irqs)(pic, is->is_irq & ~0x1f,
504 	    __BIT(is->is_irq & 0x1f));
505 
506 	/* We're done. */
507 	return is;
508 }
509 
510 void
511 pic_disestablish_source(struct intrsource *is)
512 {
513 	struct pic_softc * const pic = is->is_pic;
514 	const int irq = is->is_irq;
515 
516 	(*pic->pic_ops->pic_block_irqs)(pic, irq & ~31, __BIT(irq));
517 	pic->pic_sources[irq] = NULL;
518 	pic__iplsources[pic_ipl_offset[is->is_ipl] + is->is_iplidx] = NULL;
519 	evcnt_detach(&is->is_ev);
520 
521 	free(is, M_INTRSOURCE);
522 }
523 
524 int
525 _splraise(int newipl)
526 {
527 	struct cpu_info * const ci = curcpu();
528 	const int oldipl = ci->ci_cpl;
529 	KASSERT(newipl < NIPL);
530 	if (newipl > ci->ci_cpl)
531 		ci->ci_cpl = newipl;
532 	return oldipl;
533 }
534 int
535 _spllower(int newipl)
536 {
537 	struct cpu_info * const ci = curcpu();
538 	const int oldipl = ci->ci_cpl;
539 	KASSERT(panicstr || newipl <= ci->ci_cpl);
540 	if (newipl < ci->ci_cpl) {
541 		register_t psw = disable_interrupts(I32_bit);
542 		pic_do_pending_ints(psw, newipl, NULL);
543 		restore_interrupts(psw);
544 	}
545 	return oldipl;
546 }
547 
548 void
549 splx(int savedipl)
550 {
551 	struct cpu_info * const ci = curcpu();
552 	KASSERT(savedipl < NIPL);
553 	if (savedipl < ci->ci_cpl) {
554 		register_t psw = disable_interrupts(I32_bit);
555 		pic_do_pending_ints(psw, savedipl, NULL);
556 		restore_interrupts(psw);
557 	}
558 	ci->ci_cpl = savedipl;
559 }
560 
561 void *
562 intr_establish(int irq, int ipl, int type, int (*func)(void *), void *arg)
563 {
564 	int slot;
565 
566 	for (slot = 0; slot < PIC_MAXPICS; slot++) {
567 		struct pic_softc * const pic = pic_list[slot];
568 		if (pic == NULL || pic->pic_irqbase < 0)
569 			continue;
570 		if (pic->pic_irqbase <= irq
571 		    && irq < pic->pic_irqbase + pic->pic_maxsources) {
572 			return pic_establish_intr(pic, irq - pic->pic_irqbase,
573 			    ipl, type, func, arg);
574 		}
575 	}
576 
577 	return NULL;
578 }
579 
580 void
581 intr_disestablish(void *ih)
582 {
583 	struct intrsource * const is = ih;
584 	pic_disestablish_source(is);
585 }
586