xref: /netbsd-src/sys/arch/arm/pic/pic.c (revision 4b71a66d0f279143147d63ebfcfd8a59499a3684) 
1 /*	$NetBSD: pic.c,v 1.3 2008/04/28 20:23:14 martin Exp $	*/
2 /*-
3  * Copyright (c) 2008 The NetBSD Foundation, Inc.
4  * All rights reserved.
5  *
6  * This code is derived from software contributed to The NetBSD Foundation
7  * by Matt Thomas.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
19  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
20  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
22  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28  * POSSIBILITY OF SUCH DAMAGE.
29  */
30 #include <sys/cdefs.h>
31 __KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.3 2008/04/28 20:23:14 martin Exp $");
32 
33 #define _INTR_PRIVATE
34 #include <sys/param.h>
35 #include <sys/evcnt.h>
36 #include <sys/atomic.h>
37 #include <sys/malloc.h>
38 #include <sys/mallocvar.h>
39 
40 #include <arm/armreg.h>
41 #include <arm/cpu.h>
42 #include <arm/cpufunc.h>
43 
44 #include <arm/pic/picvar.h>
45 
46 MALLOC_DEFINE(M_INTRSOURCE, "intrsource", "interrupt source");
47 
48 static uint32_t
49 	pic_find_pending_irqs_by_ipl(struct pic_softc *, size_t, uint32_t, int);
50 static struct pic_softc *
51 	pic_list_find_pic_by_pending_ipl(uint32_t);
52 static void
53 	pic_deliver_irqs(struct pic_softc *, int, void *);
54 static void
55 	pic_list_deliver_irqs(register_t, int, void *);
56 
57 struct pic_softc *pic_list[PIC_MAXPICS];
58 #if PIC_MAXPICS > 32
59 #error PIC_MAXPICS > 32 not supported
60 #endif
61 volatile uint32_t pic_blocked_pics;
62 volatile uint32_t pic_pending_pics;
63 volatile uint32_t pic_pending_ipls;
64 volatile uint32_t pic_pending_ipl_refcnt[NIPL];
65 struct intrsource *pic_sources[PIC_MAXMAXSOURCES];
66 struct intrsource *pic__iplsources[PIC_MAXMAXSOURCES];
67 struct intrsource **pic_iplsource[NIPL] = {
68 	[0 ... NIPL-1] = pic__iplsources,
69 };
70 size_t pic_ipl_offset[NIPL+1];
71 size_t pic_sourcebase;
72 static struct evcnt pic_deferral_ev =
73     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "deferred", "intr");
74 EVCNT_ATTACH_STATIC(pic_deferral_ev);
75 
76 
77 
78 int
79 pic_handle_intr(void *arg)
80 {
81 	struct pic_softc * const pic = arg;
82 	int rv;
83 
84 	rv = (*pic->pic_ops->pic_find_pending_irqs)(pic);
85 
86 	return rv > 0;
87 }
88 
89 void
90 pic_mark_pending_source(struct pic_softc *pic, struct intrsource *is)
91 {
92 	const uint32_t ipl_mask = __BIT(is->is_ipl);
93 
94 	pic->pic_pending_irqs[is->is_irq >> 5] |= __BIT(is->is_irq & 0x1f);
95 
96 	pic->pic_pending_ipls |= ipl_mask;
97 	pic_pending_ipls |= ipl_mask;
98 	pic_pending_pics |= __BIT(pic->pic_id);
99 }
100 
101 void
102 pic_mark_pending(struct pic_softc *pic, int irq)
103 {
104 	struct intrsource * const is = pic->pic_sources[irq];
105 
106 	KASSERT(irq < pic->pic_maxsources);
107 	KASSERT(is != NULL);
108 
109 	pic_mark_pending_source(pic, is);
110 }
111 
112 uint32_t
113 pic_mark_pending_sources(struct pic_softc *pic, size_t irq_base,
114 	uint32_t pending)
115 {
116 	struct intrsource ** const isbase = &pic->pic_sources[irq_base];
117 	struct intrsource *is;
118 	uint32_t *ipending = &pic->pic_pending_irqs[irq_base >> 5];
119 	uint32_t ipl_mask = 0;
120 
121 	if (pending == 0)
122 		return ipl_mask;
123 
124 	KASSERT((irq_base & 31) == 0);
125 
126 	(*pic->pic_ops->pic_block_irqs)(pic, irq_base, pending);
127 
128 	*ipending |= pending;
129         while (pending != 0) {
130 		int n = ffs(pending);
131 		if (n-- == 0)
132 			break;
133 		is = isbase[n];
134 		KASSERT(is != NULL);
135 		KASSERT(irq_base <= is->is_irq && is->is_irq < irq_base + 32);
136 		pending &= ~__BIT(n);
137 		ipl_mask |= __BIT(is->is_ipl);
138 	}
139 
140 	pic->pic_pending_ipls |= ipl_mask;
141 	pic_pending_ipls |= ipl_mask;
142 	pic_pending_pics |= __BIT(pic->pic_id);
143 
144 	return ipl_mask;
145 }
146 
147 uint32_t
148 pic_find_pending_irqs_by_ipl(struct pic_softc *pic, size_t irq_base,
149 	uint32_t pending, int ipl)
150 {
151 	uint32_t ipl_irq_mask = 0;
152 	uint32_t irq_mask;
153 
154 	for (;;) {
155 		int irq = ffs(pending);
156 		if (irq-- == 0)
157 			return ipl_irq_mask;
158 
159 		irq_mask = __BIT(irq);
160 		KASSERT(pic->pic_sources[irq_base + irq] != NULL);
161 		if (pic->pic_sources[irq_base + irq]->is_ipl == ipl)
162 			ipl_irq_mask |= irq_mask;
163 
164 		pending &= ~irq_mask;
165 	}
166 }
167 
168 void
169 pic_dispatch(struct intrsource *is, void *frame)
170 {
171 	int rv;
172 
173 	if (__predict_false(is->is_arg == NULL)
174 	    && __predict_true(frame != NULL)) {
175 		rv = (*is->is_func)(frame);
176 	} else if (__predict_true(is->is_arg != NULL)) {
177 		rv = (*is->is_func)(is->is_arg);
178 	} else {
179 		pic_deferral_ev.ev_count++;
180 		return;
181 	}
182 	is->is_ev.ev_count++;
183 }
184 
185 void
186 pic_deliver_irqs(struct pic_softc *pic, int ipl, void *frame)
187 {
188 	const uint32_t ipl_mask = __BIT(ipl);
189 	struct intrsource *is;
190 	uint32_t *ipending = pic->pic_pending_irqs;
191 	uint32_t *iblocked = pic->pic_blocked_irqs;
192 	size_t irq_base;
193 #if PIC_MAXSOURCES > 32
194 	size_t irq_count;
195 #endif
196 	uint32_t pending_irqs;
197 	uint32_t blocked_irqs;
198 	int irq;
199 	bool progress = false;
200 
201 	KASSERT(pic->pic_pending_ipls & ipl_mask);
202 
203 	irq_base = 0;
204 #if PIC_MAXSOURCES > 32
205 	irq_count = 0;
206 #endif
207 
208 	for (;;) {
209 		pending_irqs = pic_find_pending_irqs_by_ipl(pic, irq_base,
210 		    *ipending, ipl);
211 		KASSERT((pending_irqs & *ipending) == pending_irqs);
212 		KASSERT((pending_irqs & ~(*ipending)) == 0);
213 		if (pending_irqs == 0) {
214 #if PIC_MAXSOURCES > 32
215 			irq_count += 32;
216 			if (__predict_true(irq_count >= pic->pic_maxsources))
217 				break;
218 			irq_base += 32;
219 			ipending++;
220 			iblocked++;
221 			if (irq_base >= pic->pic_maxsources) {
222 				ipending = pic->pic_pending_irqs;
223 				iblocked = pic->pic_blocked_irqs;
224 			}
225 			continue;
226 #else
227 			break;
228 #endif
229 		}
230 		progress = true;
231 		blocked_irqs = pending_irqs;
232 		do {
233 			irq = ffs(pending_irqs) - 1;
234 			KASSERT(irq >= 0);
235 
236 			*ipending &= ~__BIT(irq);
237 			is = pic->pic_sources[irq_base + irq];
238 			if (is != NULL) {
239 				cpsie(I32_bit);
240 				pic_dispatch(is, frame);
241 				cpsid(I32_bit);
242 			} else {
243 				KASSERT(0);
244 				blocked_irqs &= ~__BIT(irq);
245 			}
246 			pending_irqs = pic_find_pending_irqs_by_ipl(pic,
247 			    irq_base, *ipending, ipl);
248 		} while (pending_irqs);
249 		if (blocked_irqs) {
250 			*iblocked |= blocked_irqs;
251 			pic_blocked_pics |= __BIT(pic->pic_id);
252 		}
253 	}
254 
255 	KASSERT(progress);
256 	/*
257 	 * Since interrupts are disabled, we don't have to be too careful
258 	 * about these.
259 	 */
260 	pic->pic_pending_ipls &= ~ipl_mask;
261 	if (pic->pic_pending_ipls == 0)
262 		pic_pending_pics &= ~__BIT(pic->pic_id);
263 }
264 
265 static void
266 pic_list_unblock_irqs(void)
267 {
268 	uint32_t blocked = pic_blocked_pics;
269 
270 	pic_blocked_pics = 0;
271 	for (;;) {
272 		struct pic_softc *pic;
273 #if PIC_MAXSOURCES > 32
274 		uint32_t *iblocked;
275 		size_t irq_base;
276 #endif
277 
278 		int pic_id = ffs(blocked);
279 		if (pic_id-- == 0)
280 			return;
281 
282 		pic = pic_list[pic_id];
283 		KASSERT(pic != NULL);
284 #if PIC_MAXSOURCES > 32
285 		for (irq_base = 0, iblocked = pic->pic_blocked_irqs;
286 		     irq_base < pic->pic_maxsources;
287 		     irq_base += 32, iblocked++) {
288 			if (*iblocked) {
289 				(*pic->pic_ops->pic_unblock_irqs)(pic,
290 				    irq_base, *iblocked);
291 				*iblocked = 0;
292 			}
293 		}
294 		blocked &= ~__BIT(pic_id);
295 #else
296 		KASSERT(pic->pic_blocked_irqs[0] != 0);
297 		(*pic->pic_ops->pic_unblock_irqs)(pic,
298 		    0, pic->pic_blocked_irqs[0]);
299 #endif
300 	}
301 }
302 
303 
304 struct pic_softc *
305 pic_list_find_pic_by_pending_ipl(uint32_t ipl_mask)
306 {
307 	uint32_t pending = pic_pending_pics;
308 	struct pic_softc *pic;
309 
310 	for (;;) {
311 		int pic_id = ffs(pending);
312 		if (pic_id-- == 0)
313 			return NULL;
314 
315 		pic = pic_list[pic_id];
316 		KASSERT(pic != NULL);
317 		if (pic->pic_pending_ipls & ipl_mask)
318 			return pic;
319 		pending &= ~__BIT(pic_id);
320 	}
321 }
322 
323 void
324 pic_list_deliver_irqs(register_t psw, int ipl, void *frame)
325 {
326 	const uint32_t ipl_mask = __BIT(ipl);
327 	struct pic_softc *pic;
328 
329 	while ((pic = pic_list_find_pic_by_pending_ipl(ipl_mask)) != NULL) {
330 		pic_deliver_irqs(pic, ipl, frame);
331 		KASSERT((pic->pic_pending_ipls & ipl_mask) == 0);
332 	}
333 	pic_pending_ipls &= ~ipl_mask;
334 }
335 
336 void
337 pic_do_pending_ints(register_t psw, int newipl, void *frame)
338 {
339 	struct cpu_info * const ci = curcpu();
340 	if (__predict_false(newipl == IPL_HIGH))
341 		return;
342 	while ((pic_pending_ipls & ~__BIT(newipl)) > __BIT(newipl)) {
343 		KASSERT(pic_pending_ipls < __BIT(NIPL));
344 		for (;;) {
345 			int ipl = 31 - __builtin_clz(pic_pending_ipls);
346 			KASSERT(ipl < NIPL);
347 			if (ipl <= newipl)
348 				break;
349 
350 			ci->ci_cpl = ipl;
351 			pic_list_deliver_irqs(psw, ipl, frame);
352 			pic_list_unblock_irqs();
353 		}
354 	}
355 	if (ci->ci_cpl != newipl)
356 		ci->ci_cpl = newipl;
357 #ifdef __HAVE_FAST_SOFTINTS
358 	cpu_dosoftints();
359 #endif
360 }
361 
362 void
363 pic_add(struct pic_softc *pic, int irqbase)
364 {
365 	int slot, maybe_slot = -1;
366 
367 	for (slot = 0; slot < PIC_MAXPICS; slot++) {
368 		struct pic_softc * const xpic = pic_list[slot];
369 		if (xpic == NULL) {
370 			if (maybe_slot < 0)
371 				maybe_slot = slot;
372 			if (irqbase < 0)
373 				break;
374 			continue;
375 		}
376 		if (irqbase < 0 || xpic->pic_irqbase < 0)
377 			continue;
378 		if (irqbase >= xpic->pic_irqbase + xpic->pic_maxsources)
379 			continue;
380 		if (irqbase + pic->pic_maxsources <= xpic->pic_irqbase)
381 			continue;
382 		panic("pic_add: pic %s (%zu sources @ irq %u) conflicts"
383 		    " with pic %s (%zu sources @ irq %u)",
384 		    pic->pic_name, pic->pic_maxsources, irqbase,
385 		    xpic->pic_name, xpic->pic_maxsources, xpic->pic_irqbase);
386 	}
387 	slot = maybe_slot;
388 #if 0
389 	printf("%s: pic_sourcebase=%zu pic_maxsources=%zu\n",
390 	    pic->pic_name, pic_sourcebase, pic->pic_maxsources);
391 #endif
392 	KASSERT(pic_sourcebase + pic->pic_maxsources <= PIC_MAXMAXSOURCES);
393 
394 	pic->pic_sources = &pic_sources[pic_sourcebase];
395 	pic->pic_irqbase = irqbase;
396 	pic_sourcebase += pic->pic_maxsources;
397 	pic->pic_id = slot;
398 	pic_list[slot] = pic;
399 }
400 
401 int
402 pic_alloc_irq(struct pic_softc *pic)
403 {
404 	int irq;
405 
406 	for (irq = 0; irq < pic->pic_maxsources; irq++) {
407 		if (pic->pic_sources[irq] == NULL)
408 			return irq;
409 	}
410 
411 	return -1;
412 }
413 
414 void *
415 pic_establish_intr(struct pic_softc *pic, int irq, int ipl, int type,
416 	int (*func)(void *), void *arg)
417 {
418 	struct intrsource *is;
419 	int off, nipl;
420 
421 	if (pic->pic_sources[irq]) {
422 		printf("pic_establish_intr: pic %s irq %d already present\n",
423 		    pic->pic_name, irq);
424 		return NULL;
425 	}
426 
427 	is = malloc(sizeof(*is), M_INTRSOURCE, M_NOWAIT|M_ZERO);
428 	if (is == NULL)
429 		return NULL;
430 
431 	is->is_pic = pic;
432 	is->is_irq = irq;
433 	is->is_ipl = ipl;
434 	is->is_type = type;
435 	is->is_func = func;
436 	is->is_arg = arg;
437 
438 	if (pic->pic_ops->pic_source_name)
439 		(*pic->pic_ops->pic_source_name)(pic, irq, is->is_source,
440 		    sizeof(is->is_source));
441 	else
442 		snprintf(is->is_source, sizeof(is->is_source), "irq %d", irq);
443 
444 	evcnt_attach_dynamic(&is->is_ev, EVCNT_TYPE_INTR, NULL,
445 	    pic->pic_name, is->is_source);
446 
447 	pic->pic_sources[irq] = is;
448 
449 	/*
450 	 * First try to use an existing slot which is empty.
451 	 */
452 	for (off = pic_ipl_offset[ipl]; off < pic_ipl_offset[ipl+1]; off++) {
453 		if (pic__iplsources[off] == NULL) {
454 			is->is_iplidx = off - pic_ipl_offset[ipl];
455 			pic__iplsources[off] = is;
456 			return is;
457 		}
458 	}
459 
460 	/*
461 	 * Move up all the sources by one.
462  	 */
463 	if (ipl < NIPL) {
464 		off = pic_ipl_offset[ipl+1];
465 		memmove(&pic__iplsources[off+1], &pic__iplsources[off],
466 		    sizeof(pic__iplsources[0]) * (pic_ipl_offset[NIPL] - off));
467 	}
468 
469 	/*
470 	 * Advance the offset of all IPLs higher than this.  Include an
471 	 * extra one as well.  Thus the number of sources per ipl is
472 	 * pic_ipl_offset[ipl+1] - pic_ipl_offset[ipl].
473 	 */
474 	for (nipl = ipl + 1; nipl <= NIPL; nipl++)
475 		pic_ipl_offset[nipl]++;
476 
477 	/*
478 	 * Insert into the previously made position at the end of this IPL's
479 	 * sources.
480 	 */
481 	off = pic_ipl_offset[ipl + 1] - 1;
482 	is->is_iplidx = off - pic_ipl_offset[ipl];
483 	pic__iplsources[off] = is;
484 
485 	(*pic->pic_ops->pic_establish_irq)(pic, is);
486 
487 	(*pic->pic_ops->pic_unblock_irqs)(pic, is->is_irq & ~0x1f,
488 	    __BIT(is->is_irq & 0x1f));
489 
490 	/* We're done. */
491 	return is;
492 }
493 
494 void
495 pic_disestablish_source(struct intrsource *is)
496 {
497 	struct pic_softc * const pic = is->is_pic;
498 	const int irq = is->is_irq;
499 
500 	(*pic->pic_ops->pic_block_irqs)(pic, irq & ~31, __BIT(irq));
501 	pic->pic_sources[irq] = NULL;
502 	pic__iplsources[pic_ipl_offset[is->is_ipl] + is->is_iplidx] = NULL;
503 	evcnt_detach(&is->is_ev);
504 
505 	free(is, M_INTRSOURCE);
506 }
507 
508 int
509 _splraise(int newipl)
510 {
511 	struct cpu_info * const ci = curcpu();
512 	const int oldipl = ci->ci_cpl;
513 	KASSERT(newipl < NIPL);
514 	if (newipl > ci->ci_cpl)
515 		ci->ci_cpl = newipl;
516 	return oldipl;
517 }
518 int
519 _spllower(int newipl)
520 {
521 	struct cpu_info * const ci = curcpu();
522 	const int oldipl = ci->ci_cpl;
523 	KASSERT(panicstr || newipl <= ci->ci_cpl);
524 	if (newipl < ci->ci_cpl) {
525 		register_t psw = disable_interrupts(I32_bit);
526 		pic_do_pending_ints(psw, newipl, NULL);
527 		restore_interrupts(psw);
528 	}
529 	return oldipl;
530 }
531 
532 void
533 splx(int savedipl)
534 {
535 	struct cpu_info * const ci = curcpu();
536 	KASSERT(savedipl < NIPL);
537 	if (savedipl < ci->ci_cpl) {
538 		register_t psw = disable_interrupts(I32_bit);
539 		pic_do_pending_ints(psw, savedipl, NULL);
540 		restore_interrupts(psw);
541 	}
542 	ci->ci_cpl = savedipl;
543 }
544 
545 void *
546 intr_establish(int irq, int ipl, int type, int (*func)(void *), void *arg)
547 {
548 	int slot;
549 
550 	for (slot = 0; slot < PIC_MAXPICS; slot++) {
551 		struct pic_softc * const pic = pic_list[slot];
552 		if (pic == NULL || pic->pic_irqbase < 0)
553 			continue;
554 		if (pic->pic_irqbase <= irq
555 		    && irq < pic->pic_irqbase + pic->pic_maxsources) {
556 			return pic_establish_intr(pic, irq - pic->pic_irqbase,
557 			    ipl, type, func, arg);
558 		}
559 	}
560 
561 	return NULL;
562 }
563 
564 void
565 intr_disestablish(void *ih)
566 {
567 	struct intrsource * const is = ih;
568 	pic_disestablish_source(is);
569 }
570