xref: /dflybsd-src/sys/platform/pc64/apic/ioapic_abi.c (revision fda7d3889b1114d34ad3a52a7257a2b80fe24e4c)
1 /*
2  * Copyright (c) 1991 The Regents of the University of California.
3  * Copyright (c) 1996, by Steve Passe.  All rights reserved.
4  * Copyright (c) 2005,2008 The DragonFly Project.  All rights reserved.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The DragonFly Project
8  * by Matthew Dillon <dillon@backplane.com>
9  *
10  * This code is derived from software contributed to Berkeley by
11  * William Jolitz.
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  *
17  * 1. Redistributions of source code must retain the above copyright
18  *    notice, this list of conditions and the following disclaimer.
19  * 2. Redistributions in binary form must reproduce the above copyright
20  *    notice, this list of conditions and the following disclaimer in
21  *    the documentation and/or other materials provided with the
22  *    distribution.
23  * 3. Neither the name of The DragonFly Project nor the names of its
24  *    contributors may be used to endorse or promote products derived
25  *    from this software without specific, prior written permission.
26  *
27  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
28  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
29  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
30  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
31  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
32  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
33  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
34  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
35  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
36  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
37  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38  * SUCH DAMAGE.
39  */
40 
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #include <sys/kernel.h>
44 #include <sys/machintr.h>
45 #include <sys/interrupt.h>
46 #include <sys/bus.h>
47 #include <sys/rman.h>
48 #include <sys/thread2.h>
49 
50 #include <machine/smp.h>
51 #include <machine/segments.h>
52 #include <machine/md_var.h>
53 #include <machine/intr_machdep.h>
54 #include <machine/globaldata.h>
55 #include <machine/msi_var.h>
56 
57 #include <machine_base/isa/isa_intr.h>
58 #include <machine_base/icu/icu.h>
59 #include <machine_base/icu/icu_var.h>
60 #include <machine_base/apic/ioapic.h>
61 #include <machine_base/apic/ioapic_abi.h>
62 #include <machine_base/apic/ioapic_ipl.h>
63 #include <machine_base/apic/apicreg.h>
64 
65 #include <dev/acpica5/acpi_sci_var.h>
66 
67 #define IOAPIC_HWI_VECTORS	IDT_HWI_VECTORS
68 
69 extern inthand_t
70 	IDTVEC(ioapic_intr0),
71 	IDTVEC(ioapic_intr1),
72 	IDTVEC(ioapic_intr2),
73 	IDTVEC(ioapic_intr3),
74 	IDTVEC(ioapic_intr4),
75 	IDTVEC(ioapic_intr5),
76 	IDTVEC(ioapic_intr6),
77 	IDTVEC(ioapic_intr7),
78 	IDTVEC(ioapic_intr8),
79 	IDTVEC(ioapic_intr9),
80 	IDTVEC(ioapic_intr10),
81 	IDTVEC(ioapic_intr11),
82 	IDTVEC(ioapic_intr12),
83 	IDTVEC(ioapic_intr13),
84 	IDTVEC(ioapic_intr14),
85 	IDTVEC(ioapic_intr15),
86 	IDTVEC(ioapic_intr16),
87 	IDTVEC(ioapic_intr17),
88 	IDTVEC(ioapic_intr18),
89 	IDTVEC(ioapic_intr19),
90 	IDTVEC(ioapic_intr20),
91 	IDTVEC(ioapic_intr21),
92 	IDTVEC(ioapic_intr22),
93 	IDTVEC(ioapic_intr23),
94 	IDTVEC(ioapic_intr24),
95 	IDTVEC(ioapic_intr25),
96 	IDTVEC(ioapic_intr26),
97 	IDTVEC(ioapic_intr27),
98 	IDTVEC(ioapic_intr28),
99 	IDTVEC(ioapic_intr29),
100 	IDTVEC(ioapic_intr30),
101 	IDTVEC(ioapic_intr31),
102 	IDTVEC(ioapic_intr32),
103 	IDTVEC(ioapic_intr33),
104 	IDTVEC(ioapic_intr34),
105 	IDTVEC(ioapic_intr35),
106 	IDTVEC(ioapic_intr36),
107 	IDTVEC(ioapic_intr37),
108 	IDTVEC(ioapic_intr38),
109 	IDTVEC(ioapic_intr39),
110 	IDTVEC(ioapic_intr40),
111 	IDTVEC(ioapic_intr41),
112 	IDTVEC(ioapic_intr42),
113 	IDTVEC(ioapic_intr43),
114 	IDTVEC(ioapic_intr44),
115 	IDTVEC(ioapic_intr45),
116 	IDTVEC(ioapic_intr46),
117 	IDTVEC(ioapic_intr47),
118 	IDTVEC(ioapic_intr48),
119 	IDTVEC(ioapic_intr49),
120 	IDTVEC(ioapic_intr50),
121 	IDTVEC(ioapic_intr51),
122 	IDTVEC(ioapic_intr52),
123 	IDTVEC(ioapic_intr53),
124 	IDTVEC(ioapic_intr54),
125 	IDTVEC(ioapic_intr55),
126 	IDTVEC(ioapic_intr56),
127 	IDTVEC(ioapic_intr57),
128 	IDTVEC(ioapic_intr58),
129 	IDTVEC(ioapic_intr59),
130 	IDTVEC(ioapic_intr60),
131 	IDTVEC(ioapic_intr61),
132 	IDTVEC(ioapic_intr62),
133 	IDTVEC(ioapic_intr63),
134 	IDTVEC(ioapic_intr64),
135 	IDTVEC(ioapic_intr65),
136 	IDTVEC(ioapic_intr66),
137 	IDTVEC(ioapic_intr67),
138 	IDTVEC(ioapic_intr68),
139 	IDTVEC(ioapic_intr69),
140 	IDTVEC(ioapic_intr70),
141 	IDTVEC(ioapic_intr71),
142 	IDTVEC(ioapic_intr72),
143 	IDTVEC(ioapic_intr73),
144 	IDTVEC(ioapic_intr74),
145 	IDTVEC(ioapic_intr75),
146 	IDTVEC(ioapic_intr76),
147 	IDTVEC(ioapic_intr77),
148 	IDTVEC(ioapic_intr78),
149 	IDTVEC(ioapic_intr79),
150 	IDTVEC(ioapic_intr80),
151 	IDTVEC(ioapic_intr81),
152 	IDTVEC(ioapic_intr82),
153 	IDTVEC(ioapic_intr83),
154 	IDTVEC(ioapic_intr84),
155 	IDTVEC(ioapic_intr85),
156 	IDTVEC(ioapic_intr86),
157 	IDTVEC(ioapic_intr87),
158 	IDTVEC(ioapic_intr88),
159 	IDTVEC(ioapic_intr89),
160 	IDTVEC(ioapic_intr90),
161 	IDTVEC(ioapic_intr91),
162 	IDTVEC(ioapic_intr92),
163 	IDTVEC(ioapic_intr93),
164 	IDTVEC(ioapic_intr94),
165 	IDTVEC(ioapic_intr95),
166 	IDTVEC(ioapic_intr96),
167 	IDTVEC(ioapic_intr97),
168 	IDTVEC(ioapic_intr98),
169 	IDTVEC(ioapic_intr99),
170 	IDTVEC(ioapic_intr100),
171 	IDTVEC(ioapic_intr101),
172 	IDTVEC(ioapic_intr102),
173 	IDTVEC(ioapic_intr103),
174 	IDTVEC(ioapic_intr104),
175 	IDTVEC(ioapic_intr105),
176 	IDTVEC(ioapic_intr106),
177 	IDTVEC(ioapic_intr107),
178 	IDTVEC(ioapic_intr108),
179 	IDTVEC(ioapic_intr109),
180 	IDTVEC(ioapic_intr110),
181 	IDTVEC(ioapic_intr111),
182 	IDTVEC(ioapic_intr112),
183 	IDTVEC(ioapic_intr113),
184 	IDTVEC(ioapic_intr114),
185 	IDTVEC(ioapic_intr115),
186 	IDTVEC(ioapic_intr116),
187 	IDTVEC(ioapic_intr117),
188 	IDTVEC(ioapic_intr118),
189 	IDTVEC(ioapic_intr119),
190 	IDTVEC(ioapic_intr120),
191 	IDTVEC(ioapic_intr121),
192 	IDTVEC(ioapic_intr122),
193 	IDTVEC(ioapic_intr123),
194 	IDTVEC(ioapic_intr124),
195 	IDTVEC(ioapic_intr125),
196 	IDTVEC(ioapic_intr126),
197 	IDTVEC(ioapic_intr127),
198 	IDTVEC(ioapic_intr128),
199 	IDTVEC(ioapic_intr129),
200 	IDTVEC(ioapic_intr130),
201 	IDTVEC(ioapic_intr131),
202 	IDTVEC(ioapic_intr132),
203 	IDTVEC(ioapic_intr133),
204 	IDTVEC(ioapic_intr134),
205 	IDTVEC(ioapic_intr135),
206 	IDTVEC(ioapic_intr136),
207 	IDTVEC(ioapic_intr137),
208 	IDTVEC(ioapic_intr138),
209 	IDTVEC(ioapic_intr139),
210 	IDTVEC(ioapic_intr140),
211 	IDTVEC(ioapic_intr141),
212 	IDTVEC(ioapic_intr142),
213 	IDTVEC(ioapic_intr143),
214 	IDTVEC(ioapic_intr144),
215 	IDTVEC(ioapic_intr145),
216 	IDTVEC(ioapic_intr146),
217 	IDTVEC(ioapic_intr147),
218 	IDTVEC(ioapic_intr148),
219 	IDTVEC(ioapic_intr149),
220 	IDTVEC(ioapic_intr150),
221 	IDTVEC(ioapic_intr151),
222 	IDTVEC(ioapic_intr152),
223 	IDTVEC(ioapic_intr153),
224 	IDTVEC(ioapic_intr154),
225 	IDTVEC(ioapic_intr155),
226 	IDTVEC(ioapic_intr156),
227 	IDTVEC(ioapic_intr157),
228 	IDTVEC(ioapic_intr158),
229 	IDTVEC(ioapic_intr159),
230 	IDTVEC(ioapic_intr160),
231 	IDTVEC(ioapic_intr161),
232 	IDTVEC(ioapic_intr162),
233 	IDTVEC(ioapic_intr163),
234 	IDTVEC(ioapic_intr164),
235 	IDTVEC(ioapic_intr165),
236 	IDTVEC(ioapic_intr166),
237 	IDTVEC(ioapic_intr167),
238 	IDTVEC(ioapic_intr168),
239 	IDTVEC(ioapic_intr169),
240 	IDTVEC(ioapic_intr170),
241 	IDTVEC(ioapic_intr171),
242 	IDTVEC(ioapic_intr172),
243 	IDTVEC(ioapic_intr173),
244 	IDTVEC(ioapic_intr174),
245 	IDTVEC(ioapic_intr175),
246 	IDTVEC(ioapic_intr176),
247 	IDTVEC(ioapic_intr177),
248 	IDTVEC(ioapic_intr178),
249 	IDTVEC(ioapic_intr179),
250 	IDTVEC(ioapic_intr180),
251 	IDTVEC(ioapic_intr181),
252 	IDTVEC(ioapic_intr182),
253 	IDTVEC(ioapic_intr183),
254 	IDTVEC(ioapic_intr184),
255 	IDTVEC(ioapic_intr185),
256 	IDTVEC(ioapic_intr186),
257 	IDTVEC(ioapic_intr187),
258 	IDTVEC(ioapic_intr188),
259 	IDTVEC(ioapic_intr189),
260 	IDTVEC(ioapic_intr190),
261 	IDTVEC(ioapic_intr191);
262 
263 static inthand_t *ioapic_intr[IOAPIC_HWI_VECTORS] = {
264 	&IDTVEC(ioapic_intr0),
265 	&IDTVEC(ioapic_intr1),
266 	&IDTVEC(ioapic_intr2),
267 	&IDTVEC(ioapic_intr3),
268 	&IDTVEC(ioapic_intr4),
269 	&IDTVEC(ioapic_intr5),
270 	&IDTVEC(ioapic_intr6),
271 	&IDTVEC(ioapic_intr7),
272 	&IDTVEC(ioapic_intr8),
273 	&IDTVEC(ioapic_intr9),
274 	&IDTVEC(ioapic_intr10),
275 	&IDTVEC(ioapic_intr11),
276 	&IDTVEC(ioapic_intr12),
277 	&IDTVEC(ioapic_intr13),
278 	&IDTVEC(ioapic_intr14),
279 	&IDTVEC(ioapic_intr15),
280 	&IDTVEC(ioapic_intr16),
281 	&IDTVEC(ioapic_intr17),
282 	&IDTVEC(ioapic_intr18),
283 	&IDTVEC(ioapic_intr19),
284 	&IDTVEC(ioapic_intr20),
285 	&IDTVEC(ioapic_intr21),
286 	&IDTVEC(ioapic_intr22),
287 	&IDTVEC(ioapic_intr23),
288 	&IDTVEC(ioapic_intr24),
289 	&IDTVEC(ioapic_intr25),
290 	&IDTVEC(ioapic_intr26),
291 	&IDTVEC(ioapic_intr27),
292 	&IDTVEC(ioapic_intr28),
293 	&IDTVEC(ioapic_intr29),
294 	&IDTVEC(ioapic_intr30),
295 	&IDTVEC(ioapic_intr31),
296 	&IDTVEC(ioapic_intr32),
297 	&IDTVEC(ioapic_intr33),
298 	&IDTVEC(ioapic_intr34),
299 	&IDTVEC(ioapic_intr35),
300 	&IDTVEC(ioapic_intr36),
301 	&IDTVEC(ioapic_intr37),
302 	&IDTVEC(ioapic_intr38),
303 	&IDTVEC(ioapic_intr39),
304 	&IDTVEC(ioapic_intr40),
305 	&IDTVEC(ioapic_intr41),
306 	&IDTVEC(ioapic_intr42),
307 	&IDTVEC(ioapic_intr43),
308 	&IDTVEC(ioapic_intr44),
309 	&IDTVEC(ioapic_intr45),
310 	&IDTVEC(ioapic_intr46),
311 	&IDTVEC(ioapic_intr47),
312 	&IDTVEC(ioapic_intr48),
313 	&IDTVEC(ioapic_intr49),
314 	&IDTVEC(ioapic_intr50),
315 	&IDTVEC(ioapic_intr51),
316 	&IDTVEC(ioapic_intr52),
317 	&IDTVEC(ioapic_intr53),
318 	&IDTVEC(ioapic_intr54),
319 	&IDTVEC(ioapic_intr55),
320 	&IDTVEC(ioapic_intr56),
321 	&IDTVEC(ioapic_intr57),
322 	&IDTVEC(ioapic_intr58),
323 	&IDTVEC(ioapic_intr59),
324 	&IDTVEC(ioapic_intr60),
325 	&IDTVEC(ioapic_intr61),
326 	&IDTVEC(ioapic_intr62),
327 	&IDTVEC(ioapic_intr63),
328 	&IDTVEC(ioapic_intr64),
329 	&IDTVEC(ioapic_intr65),
330 	&IDTVEC(ioapic_intr66),
331 	&IDTVEC(ioapic_intr67),
332 	&IDTVEC(ioapic_intr68),
333 	&IDTVEC(ioapic_intr69),
334 	&IDTVEC(ioapic_intr70),
335 	&IDTVEC(ioapic_intr71),
336 	&IDTVEC(ioapic_intr72),
337 	&IDTVEC(ioapic_intr73),
338 	&IDTVEC(ioapic_intr74),
339 	&IDTVEC(ioapic_intr75),
340 	&IDTVEC(ioapic_intr76),
341 	&IDTVEC(ioapic_intr77),
342 	&IDTVEC(ioapic_intr78),
343 	&IDTVEC(ioapic_intr79),
344 	&IDTVEC(ioapic_intr80),
345 	&IDTVEC(ioapic_intr81),
346 	&IDTVEC(ioapic_intr82),
347 	&IDTVEC(ioapic_intr83),
348 	&IDTVEC(ioapic_intr84),
349 	&IDTVEC(ioapic_intr85),
350 	&IDTVEC(ioapic_intr86),
351 	&IDTVEC(ioapic_intr87),
352 	&IDTVEC(ioapic_intr88),
353 	&IDTVEC(ioapic_intr89),
354 	&IDTVEC(ioapic_intr90),
355 	&IDTVEC(ioapic_intr91),
356 	&IDTVEC(ioapic_intr92),
357 	&IDTVEC(ioapic_intr93),
358 	&IDTVEC(ioapic_intr94),
359 	&IDTVEC(ioapic_intr95),
360 	&IDTVEC(ioapic_intr96),
361 	&IDTVEC(ioapic_intr97),
362 	&IDTVEC(ioapic_intr98),
363 	&IDTVEC(ioapic_intr99),
364 	&IDTVEC(ioapic_intr100),
365 	&IDTVEC(ioapic_intr101),
366 	&IDTVEC(ioapic_intr102),
367 	&IDTVEC(ioapic_intr103),
368 	&IDTVEC(ioapic_intr104),
369 	&IDTVEC(ioapic_intr105),
370 	&IDTVEC(ioapic_intr106),
371 	&IDTVEC(ioapic_intr107),
372 	&IDTVEC(ioapic_intr108),
373 	&IDTVEC(ioapic_intr109),
374 	&IDTVEC(ioapic_intr110),
375 	&IDTVEC(ioapic_intr111),
376 	&IDTVEC(ioapic_intr112),
377 	&IDTVEC(ioapic_intr113),
378 	&IDTVEC(ioapic_intr114),
379 	&IDTVEC(ioapic_intr115),
380 	&IDTVEC(ioapic_intr116),
381 	&IDTVEC(ioapic_intr117),
382 	&IDTVEC(ioapic_intr118),
383 	&IDTVEC(ioapic_intr119),
384 	&IDTVEC(ioapic_intr120),
385 	&IDTVEC(ioapic_intr121),
386 	&IDTVEC(ioapic_intr122),
387 	&IDTVEC(ioapic_intr123),
388 	&IDTVEC(ioapic_intr124),
389 	&IDTVEC(ioapic_intr125),
390 	&IDTVEC(ioapic_intr126),
391 	&IDTVEC(ioapic_intr127),
392 	&IDTVEC(ioapic_intr128),
393 	&IDTVEC(ioapic_intr129),
394 	&IDTVEC(ioapic_intr130),
395 	&IDTVEC(ioapic_intr131),
396 	&IDTVEC(ioapic_intr132),
397 	&IDTVEC(ioapic_intr133),
398 	&IDTVEC(ioapic_intr134),
399 	&IDTVEC(ioapic_intr135),
400 	&IDTVEC(ioapic_intr136),
401 	&IDTVEC(ioapic_intr137),
402 	&IDTVEC(ioapic_intr138),
403 	&IDTVEC(ioapic_intr139),
404 	&IDTVEC(ioapic_intr140),
405 	&IDTVEC(ioapic_intr141),
406 	&IDTVEC(ioapic_intr142),
407 	&IDTVEC(ioapic_intr143),
408 	&IDTVEC(ioapic_intr144),
409 	&IDTVEC(ioapic_intr145),
410 	&IDTVEC(ioapic_intr146),
411 	&IDTVEC(ioapic_intr147),
412 	&IDTVEC(ioapic_intr148),
413 	&IDTVEC(ioapic_intr149),
414 	&IDTVEC(ioapic_intr150),
415 	&IDTVEC(ioapic_intr151),
416 	&IDTVEC(ioapic_intr152),
417 	&IDTVEC(ioapic_intr153),
418 	&IDTVEC(ioapic_intr154),
419 	&IDTVEC(ioapic_intr155),
420 	&IDTVEC(ioapic_intr156),
421 	&IDTVEC(ioapic_intr157),
422 	&IDTVEC(ioapic_intr158),
423 	&IDTVEC(ioapic_intr159),
424 	&IDTVEC(ioapic_intr160),
425 	&IDTVEC(ioapic_intr161),
426 	&IDTVEC(ioapic_intr162),
427 	&IDTVEC(ioapic_intr163),
428 	&IDTVEC(ioapic_intr164),
429 	&IDTVEC(ioapic_intr165),
430 	&IDTVEC(ioapic_intr166),
431 	&IDTVEC(ioapic_intr167),
432 	&IDTVEC(ioapic_intr168),
433 	&IDTVEC(ioapic_intr169),
434 	&IDTVEC(ioapic_intr170),
435 	&IDTVEC(ioapic_intr171),
436 	&IDTVEC(ioapic_intr172),
437 	&IDTVEC(ioapic_intr173),
438 	&IDTVEC(ioapic_intr174),
439 	&IDTVEC(ioapic_intr175),
440 	&IDTVEC(ioapic_intr176),
441 	&IDTVEC(ioapic_intr177),
442 	&IDTVEC(ioapic_intr178),
443 	&IDTVEC(ioapic_intr179),
444 	&IDTVEC(ioapic_intr180),
445 	&IDTVEC(ioapic_intr181),
446 	&IDTVEC(ioapic_intr182),
447 	&IDTVEC(ioapic_intr183),
448 	&IDTVEC(ioapic_intr184),
449 	&IDTVEC(ioapic_intr185),
450 	&IDTVEC(ioapic_intr186),
451 	&IDTVEC(ioapic_intr187),
452 	&IDTVEC(ioapic_intr188),
453 	&IDTVEC(ioapic_intr189),
454 	&IDTVEC(ioapic_intr190),
455 	&IDTVEC(ioapic_intr191)
456 };
457 
458 #define IOAPIC_HWI_SYSCALL	(IDT_OFFSET_SYSCALL - IDT_OFFSET)
459 
460 static struct ioapic_irqmap {
461 	int			im_type;	/* IOAPIC_IMT_ */
462 	enum intr_trigger	im_trig;
463 	enum intr_polarity	im_pola;
464 	int			im_gsi;
465 	int			im_msi_base;
466 	uint32_t		im_flags;	/* IOAPIC_IMF_ */
467 } ioapic_irqmaps[MAXCPU][IOAPIC_HWI_VECTORS];
468 
469 static struct lwkt_token ioapic_irqmap_tok =
470 	LWKT_TOKEN_INITIALIZER(ioapic_irqmap_token);
471 
472 #define IOAPIC_IMT_UNUSED	0
473 #define IOAPIC_IMT_RESERVED	1
474 #define IOAPIC_IMT_LEGACY	2
475 #define IOAPIC_IMT_SYSCALL	3
476 #define IOAPIC_IMT_MSI		4
477 #define IOAPIC_IMT_MSIX		5
478 
479 #define IOAPIC_IMT_ISHWI(map)	((map)->im_type != IOAPIC_IMT_RESERVED && \
480 				 (map)->im_type != IOAPIC_IMT_SYSCALL)
481 
482 #define IOAPIC_IMF_CONF		0x1
483 
484 extern void	IOAPIC_INTREN(int);
485 extern void	IOAPIC_INTRDIS(int);
486 
487 extern int	imcr_present;
488 
489 static void	ioapic_abi_intr_enable(int);
490 static void	ioapic_abi_intr_disable(int);
491 static void	ioapic_abi_intr_setup(int, int);
492 static void	ioapic_abi_intr_teardown(int);
493 
494 static void	ioapic_abi_legacy_intr_config(int,
495 		    enum intr_trigger, enum intr_polarity);
496 static int	ioapic_abi_legacy_intr_cpuid(int);
497 static int	ioapic_abi_legacy_intr_find(int,
498 		    enum intr_trigger, enum intr_polarity);
499 static int	ioapic_abi_legacy_intr_find_bygsi(int,
500 		    enum intr_trigger, enum intr_polarity);
501 
502 static int	ioapic_abi_msi_alloc(int [], int, int);
503 static void	ioapic_abi_msi_release(const int [], int, int);
504 static void	ioapic_abi_msi_map(int, uint64_t *, uint32_t *, int);
505 static int	ioapic_abi_msix_alloc(int *, int);
506 static void	ioapic_abi_msix_release(int, int);
507 
508 static int	ioapic_abi_msi_alloc_intern(int, const char *,
509 		    int [], int, int);
510 static void	ioapic_abi_msi_release_intern(int, const char *,
511 		    const int [], int, int);
512 
513 static void	ioapic_abi_finalize(void);
514 static void	ioapic_abi_cleanup(void);
515 static void	ioapic_abi_setdefault(void);
516 static void	ioapic_abi_stabilize(void);
517 static void	ioapic_abi_initmap(void);
518 static void	ioapic_abi_rman_setup(struct rman *);
519 
520 static int	ioapic_abi_gsi_cpuid(int, int);
521 static int	ioapic_find_unused_irqmap(int);
522 
523 struct machintr_abi MachIntrABI_IOAPIC = {
524 	MACHINTR_IOAPIC,
525 	.intr_disable	= ioapic_abi_intr_disable,
526 	.intr_enable	= ioapic_abi_intr_enable,
527 	.intr_setup	= ioapic_abi_intr_setup,
528 	.intr_teardown	= ioapic_abi_intr_teardown,
529 
530 	.legacy_intr_config = ioapic_abi_legacy_intr_config,
531 	.legacy_intr_cpuid = ioapic_abi_legacy_intr_cpuid,
532 	.legacy_intr_find = ioapic_abi_legacy_intr_find,
533 	.legacy_intr_find_bygsi = ioapic_abi_legacy_intr_find_bygsi,
534 
535 	.msi_alloc	= ioapic_abi_msi_alloc,
536 	.msi_release	= ioapic_abi_msi_release,
537 	.msi_map	= ioapic_abi_msi_map,
538 	.msix_alloc	= ioapic_abi_msix_alloc,
539 	.msix_release	= ioapic_abi_msix_release,
540 
541 	.finalize	= ioapic_abi_finalize,
542 	.cleanup	= ioapic_abi_cleanup,
543 	.setdefault	= ioapic_abi_setdefault,
544 	.stabilize	= ioapic_abi_stabilize,
545 	.initmap	= ioapic_abi_initmap,
546 	.rman_setup	= ioapic_abi_rman_setup
547 };
548 
549 static int	ioapic_abi_extint_irq = -1;
550 static int	ioapic_abi_legacy_irq_max;
551 static int	ioapic_abi_gsi_balance;
552 static int	ioapic_abi_msi_start;	/* NOTE: for testing only */
553 
554 struct ioapic_irqinfo	ioapic_irqs[IOAPIC_HWI_VECTORS];
555 
556 static void
557 ioapic_abi_intr_enable(int irq)
558 {
559 	const struct ioapic_irqmap *map;
560 
561 	KASSERT(irq >= 0 && irq < IOAPIC_HWI_VECTORS,
562 	    ("ioapic enable, invalid irq %d", irq));
563 
564 	map = &ioapic_irqmaps[mycpuid][irq];
565 	KASSERT(IOAPIC_IMT_ISHWI(map),
566 	    ("ioapic enable, not hwi irq %d, type %d, cpu%d",
567 	     irq, map->im_type, mycpuid));
568 	if (map->im_type != IOAPIC_IMT_LEGACY)
569 		return;
570 
571 	IOAPIC_INTREN(irq);
572 }
573 
574 static void
575 ioapic_abi_intr_disable(int irq)
576 {
577 	const struct ioapic_irqmap *map;
578 
579 	KASSERT(irq >= 0 && irq < IOAPIC_HWI_VECTORS,
580 	    ("ioapic disable, invalid irq %d", irq));
581 
582 	map = &ioapic_irqmaps[mycpuid][irq];
583 	KASSERT(IOAPIC_IMT_ISHWI(map),
584 	    ("ioapic disable, not hwi irq %d, type %d, cpu%d",
585 	     irq, map->im_type, mycpuid));
586 	if (map->im_type != IOAPIC_IMT_LEGACY)
587 		return;
588 
589 	IOAPIC_INTRDIS(irq);
590 }
591 
592 static void
593 ioapic_abi_finalize(void)
594 {
595 	KKASSERT(MachIntrABI.type == MACHINTR_IOAPIC);
596 	KKASSERT(ioapic_enable);
597 
598 	/*
599 	 * If an IMCR is present, program bit 0 to disconnect the 8259
600 	 * from the BSP.
601 	 */
602 	if (imcr_present) {
603 		outb(0x22, 0x70);	/* select IMCR */
604 		outb(0x23, 0x01);	/* disconnect 8259 */
605 	}
606 }
607 
608 /*
609  * This routine is called after physical interrupts are enabled but before
610  * the critical section is released.  We need to clean out any interrupts
611  * that had already been posted to the cpu.
612  */
613 static void
614 ioapic_abi_cleanup(void)
615 {
616 	bzero(mdcpu->gd_ipending, sizeof(mdcpu->gd_ipending));
617 }
618 
619 /* Must never be called */
620 static void
621 ioapic_abi_stabilize(void)
622 {
623 	panic("ioapic_stabilize is called");
624 }
625 
626 static void
627 ioapic_abi_intr_setup(int intr, int flags)
628 {
629 	const struct ioapic_irqmap *map;
630 	int vector, select;
631 	uint32_t value;
632 	register_t ef;
633 
634 	KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS,
635 	    ("ioapic setup, invalid irq %d", intr));
636 
637 	map = &ioapic_irqmaps[mycpuid][intr];
638 	KASSERT(IOAPIC_IMT_ISHWI(map),
639 	    ("ioapic setup, not hwi irq %d, type %d, cpu%d",
640 	     intr, map->im_type, mycpuid));
641 	if (map->im_type != IOAPIC_IMT_LEGACY)
642 		return;
643 
644 	KASSERT(ioapic_irqs[intr].io_addr != NULL,
645 	    ("ioapic setup, no GSI information, irq %d", intr));
646 
647 	ef = read_rflags();
648 	cpu_disable_intr();
649 
650 	vector = IDT_OFFSET + intr;
651 
652 	/*
653 	 * Now reprogram the vector in the IO APIC.  In order to avoid
654 	 * losing an EOI for a level interrupt, which is vector based,
655 	 * make sure that the IO APIC is programmed for edge-triggering
656 	 * first, then reprogrammed with the new vector.  This should
657 	 * clear the IRR bit.
658 	 */
659 	imen_lock();
660 
661 	select = ioapic_irqs[intr].io_idx;
662 	value = ioapic_read(ioapic_irqs[intr].io_addr, select);
663 	value |= IOART_INTMSET;
664 
665 	ioapic_write(ioapic_irqs[intr].io_addr, select,
666 	    (value & ~APIC_TRIGMOD_MASK));
667 	ioapic_write(ioapic_irqs[intr].io_addr, select,
668 	    (value & ~IOART_INTVEC) | vector);
669 
670 	imen_unlock();
671 
672 	IOAPIC_INTREN(intr);
673 
674 	write_rflags(ef);
675 }
676 
677 static void
678 ioapic_abi_intr_teardown(int intr)
679 {
680 	const struct ioapic_irqmap *map;
681 	int vector, select;
682 	uint32_t value;
683 	register_t ef;
684 
685 	KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS,
686 	    ("ioapic teardown, invalid irq %d", intr));
687 
688 	map = &ioapic_irqmaps[mycpuid][intr];
689 	KASSERT(IOAPIC_IMT_ISHWI(map),
690 	    ("ioapic teardown, not hwi irq %d, type %d, cpu%d",
691 	     intr, map->im_type, mycpuid));
692 	if (map->im_type != IOAPIC_IMT_LEGACY)
693 		return;
694 
695 	KASSERT(ioapic_irqs[intr].io_addr != NULL,
696 	    ("ioapic teardown, no GSI information, irq %d", intr));
697 
698 	ef = read_rflags();
699 	cpu_disable_intr();
700 
701 	/*
702 	 * Teardown an interrupt vector.  The vector should already be
703 	 * installed in the cpu's IDT, but make sure.
704 	 */
705 	IOAPIC_INTRDIS(intr);
706 
707 	vector = IDT_OFFSET + intr;
708 
709 	/*
710 	 * In order to avoid losing an EOI for a level interrupt, which
711 	 * is vector based, make sure that the IO APIC is programmed for
712 	 * edge-triggering first, then reprogrammed with the new vector.
713 	 * This should clear the IRR bit.
714 	 */
715 	imen_lock();
716 
717 	select = ioapic_irqs[intr].io_idx;
718 	value = ioapic_read(ioapic_irqs[intr].io_addr, select);
719 
720 	ioapic_write(ioapic_irqs[intr].io_addr, select,
721 	    (value & ~APIC_TRIGMOD_MASK));
722 	ioapic_write(ioapic_irqs[intr].io_addr, select,
723 	    (value & ~IOART_INTVEC) | vector);
724 
725 	imen_unlock();
726 
727 	write_rflags(ef);
728 }
729 
730 static void
731 ioapic_abi_setdefault(void)
732 {
733 	int intr;
734 
735 	for (intr = 0; intr < IOAPIC_HWI_VECTORS; ++intr) {
736 		if (intr == IOAPIC_HWI_SYSCALL)
737 			continue;
738 		setidt_global(IDT_OFFSET + intr, ioapic_intr[intr],
739 		    SDT_SYSIGT, SEL_KPL, 0);
740 	}
741 }
742 
743 static void
744 ioapic_abi_initmap(void)
745 {
746 	int cpu;
747 
748 	kgetenv_int("hw.ioapic.gsi.balance", &ioapic_abi_gsi_balance);
749 
750 	kgetenv_int("hw.ioapic.msi_start", &ioapic_abi_msi_start);
751 	ioapic_abi_msi_start &= ~0x1f;	/* MUST be 32 aligned */
752 
753 	/*
754 	 * NOTE: ncpus is not ready yet
755 	 */
756 	for (cpu = 0; cpu < MAXCPU; ++cpu) {
757 		int i;
758 
759 		for (i = 0; i < IOAPIC_HWI_VECTORS; ++i) {
760 			ioapic_irqmaps[cpu][i].im_gsi = -1;
761 			ioapic_irqmaps[cpu][i].im_msi_base = -1;
762 		}
763 		ioapic_irqmaps[cpu][IOAPIC_HWI_SYSCALL].im_type =
764 		    IOAPIC_IMT_SYSCALL;
765 	}
766 }
767 
768 static int
769 ioapic_find_unused_irqmap(int gsi)
770 {
771 	int cpuid, i;
772 
773 	cpuid = ioapic_abi_gsi_cpuid(-1, gsi);
774 
775 	for (i = ISA_IRQ_CNT; i < IOAPIC_HWI_VECTORS; ++i) {
776 		if (i == acpi_sci_irqno())
777 			continue;
778 		if (ioapic_irqmaps[cpuid][i].im_type == IOAPIC_IMT_UNUSED)
779 			return i;
780 	}
781 	return -1;
782 }
783 
784 void
785 ioapic_set_legacy_irqmap(int irq, int gsi, enum intr_trigger trig,
786     enum intr_polarity pola)
787 {
788 	struct ioapic_irqinfo *info;
789 	struct ioapic_irqmap *map;
790 	void *ioaddr;
791 	int pin, cpuid;
792 
793 	KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL);
794 	KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW);
795 
796 	KKASSERT(irq >= 0);
797 	if (irq >= IOAPIC_HWI_VECTORS) {
798 		/*
799 		 * Some BIOSes seem to assume that all 256 IDT vectors
800 		 * could be used, while we limit the available IDT
801 		 * vectors to 192; find an unused IRQ for this GSI.
802 		 */
803 		irq = ioapic_find_unused_irqmap(gsi);
804 		if (irq < 0) {
805 			kprintf("failed to find unused irq for gsi %d\n", gsi);
806 			return;
807 		}
808 	}
809 	KKASSERT(irq < IOAPIC_HWI_VECTORS);
810 
811 	if (irq > ioapic_abi_legacy_irq_max)
812 		ioapic_abi_legacy_irq_max = irq;
813 
814 	cpuid = ioapic_abi_gsi_cpuid(irq, gsi);
815 
816 	map = &ioapic_irqmaps[cpuid][irq];
817 
818 	KKASSERT(map->im_type == IOAPIC_IMT_UNUSED);
819 	map->im_type = IOAPIC_IMT_LEGACY;
820 
821 	map->im_gsi = gsi;
822 	map->im_trig = trig;
823 	map->im_pola = pola;
824 
825 	if (bootverbose) {
826 		kprintf("IOAPIC: irq %d -> gsi %d %s/%s\n",
827 			irq, map->im_gsi,
828 			intr_str_trigger(map->im_trig),
829 			intr_str_polarity(map->im_pola));
830 	}
831 
832 	pin = ioapic_gsi_pin(map->im_gsi);
833 	ioaddr = ioapic_gsi_ioaddr(map->im_gsi);
834 
835 	info = &ioapic_irqs[irq];
836 
837 	imen_lock();
838 
839 	info->io_addr = ioaddr;
840 	info->io_idx = IOAPIC_REDTBL + (2 * pin);
841 	info->io_flags = IOAPIC_IRQI_FLAG_MASKED;
842 	if (map->im_trig == INTR_TRIGGER_LEVEL)
843 		info->io_flags |= IOAPIC_IRQI_FLAG_LEVEL;
844 
845 	ioapic_pin_setup(ioaddr, pin, IDT_OFFSET + irq,
846 	    map->im_trig, map->im_pola, cpuid);
847 
848 	imen_unlock();
849 }
850 
851 void
852 ioapic_fixup_legacy_irqmaps(void)
853 {
854 	int cpu;
855 
856 	for (cpu = 0; cpu < ncpus; ++cpu) {
857 		int i;
858 
859 		for (i = 0; i < ISA_IRQ_CNT; ++i) {
860 			struct ioapic_irqmap *map = &ioapic_irqmaps[cpu][i];
861 
862 			if (map->im_type == IOAPIC_IMT_UNUSED) {
863 				map->im_type = IOAPIC_IMT_RESERVED;
864 				if (bootverbose) {
865 					kprintf("IOAPIC: "
866 					    "cpu%d irq %d reserved\n", cpu, i);
867 				}
868 			}
869 		}
870 	}
871 
872 	ioapic_abi_legacy_irq_max += 1;
873 	if (bootverbose) {
874 		kprintf("IOAPIC: legacy irq max %d\n",
875 		    ioapic_abi_legacy_irq_max);
876 	}
877 }
878 
879 static int
880 ioapic_abi_legacy_intr_find_bygsi(int gsi, enum intr_trigger trig,
881     enum intr_polarity pola)
882 {
883 	int cpu;
884 
885 #ifdef INVARIANTS
886 	if (trig == INTR_TRIGGER_CONFORM) {
887 		KKASSERT(pola == INTR_POLARITY_CONFORM);
888 	} else {
889 		KKASSERT(trig == INTR_TRIGGER_EDGE ||
890 		    trig == INTR_TRIGGER_LEVEL);
891 		KKASSERT(pola == INTR_POLARITY_HIGH ||
892 		    pola == INTR_POLARITY_LOW);
893 	}
894 #endif
895 
896 	for (cpu = 0; cpu < ncpus; ++cpu) {
897 		int irq;
898 
899 		for (irq = 0; irq < ioapic_abi_legacy_irq_max; ++irq) {
900 			const struct ioapic_irqmap *map =
901 			    &ioapic_irqmaps[cpu][irq];
902 
903 			if (map->im_gsi == gsi) {
904 				KKASSERT(map->im_type == IOAPIC_IMT_LEGACY);
905 
906 				if ((map->im_flags & IOAPIC_IMF_CONF) &&
907 				    trig != INTR_TRIGGER_CONFORM &&
908 				    pola != INTR_POLARITY_CONFORM) {
909 					if (map->im_trig != trig ||
910 					    map->im_pola != pola)
911 						return -1;
912 				}
913 				return irq;
914 			}
915 		}
916 	}
917 	return -1;
918 }
919 
920 static int
921 ioapic_abi_legacy_intr_find(int irq, enum intr_trigger trig,
922     enum intr_polarity pola)
923 {
924 	int cpu;
925 
926 #ifdef INVARIANTS
927 	if (trig == INTR_TRIGGER_CONFORM) {
928 		KKASSERT(pola == INTR_POLARITY_CONFORM);
929 	} else {
930 		KKASSERT(trig == INTR_TRIGGER_EDGE ||
931 		    trig == INTR_TRIGGER_LEVEL);
932 		KKASSERT(pola == INTR_POLARITY_HIGH ||
933 		    pola == INTR_POLARITY_LOW);
934 	}
935 #endif
936 
937 	if (irq < 0 || irq >= ioapic_abi_legacy_irq_max)
938 		return -1;
939 
940 	for (cpu = 0; cpu < ncpus; ++cpu) {
941 		const struct ioapic_irqmap *map = &ioapic_irqmaps[cpu][irq];
942 
943 		if (map->im_type == IOAPIC_IMT_LEGACY) {
944 			if ((map->im_flags & IOAPIC_IMF_CONF) &&
945 			    trig != INTR_TRIGGER_CONFORM &&
946 			    pola != INTR_POLARITY_CONFORM) {
947 				if (map->im_trig != trig ||
948 				    map->im_pola != pola)
949 					return -1;
950 			}
951 			return irq;
952 		}
953 	}
954 	return -1;
955 }
956 
957 static void
958 ioapic_abi_legacy_intr_config(int irq, enum intr_trigger trig,
959     enum intr_polarity pola)
960 {
961 	struct ioapic_irqinfo *info;
962 	struct ioapic_irqmap *map = NULL;
963 	void *ioaddr;
964 	int pin, cpuid;
965 
966 	KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL);
967 	KKASSERT(pola == INTR_POLARITY_HIGH || pola == INTR_POLARITY_LOW);
968 
969 	KKASSERT(irq >= 0 && irq < ioapic_abi_legacy_irq_max);
970 	for (cpuid = 0; cpuid < ncpus; ++cpuid) {
971 		map = &ioapic_irqmaps[cpuid][irq];
972 		if (map->im_type == IOAPIC_IMT_LEGACY)
973 			break;
974 	}
975 	KKASSERT(cpuid < ncpus);
976 
977 #ifdef notyet
978 	if (map->im_flags & IOAPIC_IMF_CONF) {
979 		if (trig != map->im_trig) {
980 			panic("ioapic_intr_config: trig %s -> %s",
981 			      intr_str_trigger(map->im_trig),
982 			      intr_str_trigger(trig));
983 		}
984 		if (pola != map->im_pola) {
985 			panic("ioapic_intr_config: pola %s -> %s",
986 			      intr_str_polarity(map->im_pola),
987 			      intr_str_polarity(pola));
988 		}
989 		return;
990 	}
991 #endif
992 	map->im_flags |= IOAPIC_IMF_CONF;
993 
994 	if (trig == map->im_trig && pola == map->im_pola)
995 		return;
996 
997 	if (bootverbose) {
998 		kprintf("IOAPIC: irq %d, gsi %d %s/%s -> %s/%s\n",
999 			irq, map->im_gsi,
1000 			intr_str_trigger(map->im_trig),
1001 			intr_str_polarity(map->im_pola),
1002 			intr_str_trigger(trig),
1003 			intr_str_polarity(pola));
1004 	}
1005 	map->im_trig = trig;
1006 	map->im_pola = pola;
1007 
1008 	pin = ioapic_gsi_pin(map->im_gsi);
1009 	ioaddr = ioapic_gsi_ioaddr(map->im_gsi);
1010 
1011 	info = &ioapic_irqs[irq];
1012 
1013 	imen_lock();
1014 
1015 	info->io_flags &= ~IOAPIC_IRQI_FLAG_LEVEL;
1016 	if (map->im_trig == INTR_TRIGGER_LEVEL)
1017 		info->io_flags |= IOAPIC_IRQI_FLAG_LEVEL;
1018 
1019 	ioapic_pin_setup(ioaddr, pin, IDT_OFFSET + irq,
1020 	    map->im_trig, map->im_pola, cpuid);
1021 
1022 	imen_unlock();
1023 }
1024 
1025 int
1026 ioapic_conf_legacy_extint(int irq)
1027 {
1028 	struct ioapic_irqinfo *info;
1029 	struct ioapic_irqmap *map;
1030 	void *ioaddr;
1031 	int pin, error, vec;
1032 
1033 	/* XXX only irq0 is allowed */
1034 	KKASSERT(irq == 0);
1035 
1036 	vec = IDT_OFFSET + irq;
1037 
1038 	if (ioapic_abi_extint_irq == irq)
1039 		return 0;
1040 	else if (ioapic_abi_extint_irq >= 0)
1041 		return EEXIST;
1042 
1043 	error = icu_ioapic_extint(irq, vec);
1044 	if (error)
1045 		return error;
1046 
1047 	/* ExtINT is always targeted to cpu0 */
1048 	map = &ioapic_irqmaps[0][irq];
1049 
1050 	KKASSERT(map->im_type == IOAPIC_IMT_RESERVED ||
1051 		 map->im_type == IOAPIC_IMT_LEGACY);
1052 	if (map->im_type == IOAPIC_IMT_LEGACY) {
1053 		if (map->im_flags & IOAPIC_IMF_CONF)
1054 			return EEXIST;
1055 	}
1056 	ioapic_abi_extint_irq = irq;
1057 
1058 	map->im_type = IOAPIC_IMT_LEGACY;
1059 	map->im_trig = INTR_TRIGGER_EDGE;
1060 	map->im_pola = INTR_POLARITY_HIGH;
1061 	map->im_flags = IOAPIC_IMF_CONF;
1062 
1063 	map->im_gsi = ioapic_extpin_gsi();
1064 	KKASSERT(map->im_gsi >= 0);
1065 
1066 	if (bootverbose) {
1067 		kprintf("IOAPIC: irq %d -> extint gsi %d %s/%s\n",
1068 			irq, map->im_gsi,
1069 			intr_str_trigger(map->im_trig),
1070 			intr_str_polarity(map->im_pola));
1071 	}
1072 
1073 	pin = ioapic_gsi_pin(map->im_gsi);
1074 	ioaddr = ioapic_gsi_ioaddr(map->im_gsi);
1075 
1076 	info = &ioapic_irqs[irq];
1077 
1078 	imen_lock();
1079 
1080 	info->io_addr = ioaddr;
1081 	info->io_idx = IOAPIC_REDTBL + (2 * pin);
1082 	info->io_flags = IOAPIC_IRQI_FLAG_MASKED;
1083 
1084 	ioapic_extpin_setup(ioaddr, pin, vec);
1085 
1086 	imen_unlock();
1087 
1088 	return 0;
1089 }
1090 
1091 static int
1092 ioapic_abi_legacy_intr_cpuid(int irq)
1093 {
1094 	const struct ioapic_irqmap *map = NULL;
1095 	int cpuid;
1096 
1097 	KKASSERT(irq >= 0 && irq < ioapic_abi_legacy_irq_max);
1098 
1099 	for (cpuid = 0; cpuid < ncpus; ++cpuid) {
1100 		map = &ioapic_irqmaps[cpuid][irq];
1101 		if (map->im_type == IOAPIC_IMT_LEGACY)
1102 			return cpuid;
1103 	}
1104 
1105 	/* XXX some drivers tries to peek at reserved IRQs */
1106 	for (cpuid = 0; cpuid < ncpus; ++cpuid) {
1107 		map = &ioapic_irqmaps[cpuid][irq];
1108 		KKASSERT(map->im_type == IOAPIC_IMT_RESERVED);
1109 	}
1110 	return 0;
1111 }
1112 
1113 static int
1114 ioapic_abi_gsi_cpuid(int irq, int gsi)
1115 {
1116 	char envpath[32];
1117 	int cpuid = -1;
1118 
1119 	KKASSERT(gsi >= 0);
1120 
1121 	if (irq == 0 || gsi == 0) {
1122 		KKASSERT(irq >= 0);
1123 		if (bootverbose) {
1124 			kprintf("IOAPIC: irq %d, gsi %d -> cpu0 (0)\n",
1125 			    irq, gsi);
1126 		}
1127 		return 0;
1128 	}
1129 
1130 	if (irq >= 0 && irq == acpi_sci_irqno()) {
1131 		if (bootverbose) {
1132 			kprintf("IOAPIC: irq %d, gsi %d -> cpu0 (sci)\n",
1133 			    irq, gsi);
1134 		}
1135 		return 0;
1136 	}
1137 
1138 	ksnprintf(envpath, sizeof(envpath), "hw.ioapic.gsi.%d.cpu", gsi);
1139 	kgetenv_int(envpath, &cpuid);
1140 
1141 	if (cpuid < 0) {
1142 		if (!ioapic_abi_gsi_balance) {
1143 			if (irq >= 0 && bootverbose) {
1144 				kprintf("IOAPIC: irq %d, gsi %d -> cpu0 "
1145 				    "(fixed)\n", irq, gsi);
1146 			}
1147 			return 0;
1148 		}
1149 
1150 		cpuid = gsi % ncpus;
1151 		if (irq >= 0 && bootverbose) {
1152 			kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (auto)\n",
1153 			    irq, gsi, cpuid);
1154 		}
1155 	} else if (cpuid >= ncpus) {
1156 		cpuid = ncpus - 1;
1157 		if (irq >= 0 && bootverbose) {
1158 			kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (fixup)\n",
1159 			    irq, gsi, cpuid);
1160 		}
1161 	} else {
1162 		if (irq >= 0 && bootverbose) {
1163 			kprintf("IOAPIC: irq %d, gsi %d -> cpu%d (user)\n",
1164 			    irq, gsi, cpuid);
1165 		}
1166 	}
1167 	return cpuid;
1168 }
1169 
1170 static void
1171 ioapic_abi_rman_setup(struct rman *rm)
1172 {
1173 	int start, end, i;
1174 
1175 	KASSERT(rm->rm_cpuid >= 0 && rm->rm_cpuid < MAXCPU,
1176 	    ("invalid rman cpuid %d", rm->rm_cpuid));
1177 
1178 	start = end = -1;
1179 	for (i = 0; i < IOAPIC_HWI_VECTORS; ++i) {
1180 		const struct ioapic_irqmap *map =
1181 		    &ioapic_irqmaps[rm->rm_cpuid][i];
1182 
1183 		if (start < 0) {
1184 			if (IOAPIC_IMT_ISHWI(map))
1185 				start = end = i;
1186 		} else {
1187 			if (IOAPIC_IMT_ISHWI(map)) {
1188 				end = i;
1189 			} else {
1190 				KKASSERT(end >= 0);
1191 				if (bootverbose) {
1192 					kprintf("IOAPIC: rman cpu%d %d - %d\n",
1193 					    rm->rm_cpuid, start, end);
1194 				}
1195 				if (rman_manage_region(rm, start, end)) {
1196 					panic("rman_manage_region"
1197 					    "(cpu%d %d - %d)", rm->rm_cpuid,
1198 					    start, end);
1199 				}
1200 				start = end = -1;
1201 			}
1202 		}
1203 	}
1204 	if (start >= 0) {
1205 		KKASSERT(end >= 0);
1206 		if (bootverbose) {
1207 			kprintf("IOAPIC: rman cpu%d %d - %d\n",
1208 			    rm->rm_cpuid, start, end);
1209 		}
1210 		if (rman_manage_region(rm, start, end)) {
1211 			panic("rman_manage_region(cpu%d %d - %d)",
1212 			    rm->rm_cpuid, start, end);
1213 		}
1214 	}
1215 }
1216 
1217 static int
1218 ioapic_abi_msi_alloc_intern(int type, const char *desc,
1219     int intrs[], int count, int cpuid)
1220 {
1221 	int i, error;
1222 
1223 	KASSERT(cpuid >= 0 && cpuid < ncpus,
1224 	    ("invalid cpuid %d", cpuid));
1225 
1226 	KASSERT(count > 0 && count <= 32, ("invalid count %d", count));
1227 	KASSERT((count & (count - 1)) == 0,
1228 	    ("count %d is not power of 2", count));
1229 
1230 	lwkt_gettoken(&ioapic_irqmap_tok);
1231 
1232 	/*
1233 	 * NOTE:
1234 	 * Since IDT_OFFSET is 32, which is the maximum valid 'count',
1235 	 * we do not need to find out the first properly aligned
1236 	 * interrupt vector.
1237 	 */
1238 
1239 	error = EMSGSIZE;
1240 	for (i = ioapic_abi_msi_start; i < IOAPIC_HWI_VECTORS; i += count) {
1241 		int j;
1242 
1243 		if (ioapic_irqmaps[cpuid][i].im_type != IOAPIC_IMT_UNUSED)
1244 			continue;
1245 
1246 		for (j = 1; j < count; ++j) {
1247 			if (ioapic_irqmaps[cpuid][i + j].im_type !=
1248 			    IOAPIC_IMT_UNUSED)
1249 				break;
1250 		}
1251 		if (j != count)
1252 			continue;
1253 
1254 		for (j = 0; j < count; ++j) {
1255 			struct ioapic_irqmap *map;
1256 			int intr = i + j;
1257 
1258 			map = &ioapic_irqmaps[cpuid][intr];
1259 			KASSERT(map->im_msi_base < 0,
1260 			    ("intr %d, stale %s-base %d",
1261 			     intr, desc, map->im_msi_base));
1262 
1263 			map->im_type = type;
1264 			map->im_msi_base = i;
1265 
1266 			intrs[j] = intr;
1267 			msi_setup(intr, cpuid);
1268 
1269 			if (bootverbose) {
1270 				kprintf("alloc %s intr %d on cpu%d\n",
1271 				    desc, intr, cpuid);
1272 			}
1273 		}
1274 		error = 0;
1275 		break;
1276 	}
1277 
1278 	lwkt_reltoken(&ioapic_irqmap_tok);
1279 
1280 	return error;
1281 }
1282 
1283 static void
1284 ioapic_abi_msi_release_intern(int type, const char *desc,
1285     const int intrs[], int count, int cpuid)
1286 {
1287 	int i, msi_base = -1, intr_next = -1, mask;
1288 
1289 	KASSERT(cpuid >= 0 && cpuid < ncpus,
1290 	    ("invalid cpuid %d", cpuid));
1291 
1292 	KASSERT(count > 0 && count <= 32, ("invalid count %d", count));
1293 
1294 	mask = count - 1;
1295 	KASSERT((count & mask) == 0, ("count %d is not power of 2", count));
1296 
1297 	lwkt_gettoken(&ioapic_irqmap_tok);
1298 
1299 	for (i = 0; i < count; ++i) {
1300 		struct ioapic_irqmap *map;
1301 		int intr = intrs[i];
1302 
1303 		KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS,
1304 		    ("invalid intr %d", intr));
1305 
1306 		map = &ioapic_irqmaps[cpuid][intr];
1307 		KASSERT(map->im_type == type,
1308 		    ("trying to release non-%s intr %d, type %d", desc,
1309 		     intr, map->im_type));
1310 		KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr,
1311 		    ("intr %d, invalid %s-base %d", intr, desc,
1312 		     map->im_msi_base));
1313 		KASSERT((map->im_msi_base & mask) == 0,
1314 		    ("intr %d, %s-base %d is not properly aligned %d",
1315 		     intr, desc, map->im_msi_base, count));
1316 
1317 		if (msi_base < 0) {
1318 			msi_base = map->im_msi_base;
1319 		} else {
1320 			KASSERT(map->im_msi_base == msi_base,
1321 			    ("intr %d, inconsistent %s-base, "
1322 			     "was %d, now %d",
1323 			     intr, desc, msi_base, map->im_msi_base));
1324 		}
1325 
1326 		if (intr_next < intr)
1327 			intr_next = intr;
1328 
1329 		map->im_type = IOAPIC_IMT_UNUSED;
1330 		map->im_msi_base = -1;
1331 
1332 		if (bootverbose) {
1333 			kprintf("release %s intr %d on cpu%d\n",
1334 			    desc, intr, cpuid);
1335 		}
1336 	}
1337 
1338 	KKASSERT(intr_next > 0);
1339 	KKASSERT(msi_base >= 0);
1340 
1341 	++intr_next;
1342 	if (intr_next < IOAPIC_HWI_VECTORS) {
1343 		const struct ioapic_irqmap *map =
1344 		    &ioapic_irqmaps[cpuid][intr_next];
1345 
1346 		if (map->im_type == type) {
1347 			KASSERT(map->im_msi_base != msi_base,
1348 			    ("more than %d %s was allocated", count, desc));
1349 		}
1350 	}
1351 
1352 	lwkt_reltoken(&ioapic_irqmap_tok);
1353 }
1354 
1355 static int
1356 ioapic_abi_msi_alloc(int intrs[], int count, int cpuid)
1357 {
1358 	return ioapic_abi_msi_alloc_intern(IOAPIC_IMT_MSI, "MSI",
1359 	    intrs, count, cpuid);
1360 }
1361 
1362 static void
1363 ioapic_abi_msi_release(const int intrs[], int count, int cpuid)
1364 {
1365 	ioapic_abi_msi_release_intern(IOAPIC_IMT_MSI, "MSI",
1366 	    intrs, count, cpuid);
1367 }
1368 
1369 static int
1370 ioapic_abi_msix_alloc(int *intr, int cpuid)
1371 {
1372 	return ioapic_abi_msi_alloc_intern(IOAPIC_IMT_MSIX, "MSI-X",
1373 	    intr, 1, cpuid);
1374 }
1375 
1376 static void
1377 ioapic_abi_msix_release(int intr, int cpuid)
1378 {
1379 	ioapic_abi_msi_release_intern(IOAPIC_IMT_MSIX, "MSI-X",
1380 	    &intr, 1, cpuid);
1381 }
1382 
1383 static void
1384 ioapic_abi_msi_map(int intr, uint64_t *addr, uint32_t *data, int cpuid)
1385 {
1386 	const struct ioapic_irqmap *map;
1387 
1388 	KASSERT(cpuid >= 0 && cpuid < ncpus,
1389 	    ("invalid cpuid %d", cpuid));
1390 
1391 	KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS,
1392 	    ("invalid intr %d", intr));
1393 
1394 	lwkt_gettoken(&ioapic_irqmap_tok);
1395 
1396 	map = &ioapic_irqmaps[cpuid][intr];
1397 	KASSERT(map->im_type == IOAPIC_IMT_MSI ||
1398 	    map->im_type == IOAPIC_IMT_MSIX,
1399 	    ("trying to map non-MSI/MSI-X intr %d, type %d", intr, map->im_type));
1400 	KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr,
1401 	    ("intr %d, invalid %s-base %d", intr,
1402 	     map->im_type == IOAPIC_IMT_MSI ? "MSI" : "MSI-X",
1403 	     map->im_msi_base));
1404 
1405 	msi_map(map->im_msi_base, addr, data, cpuid);
1406 
1407 	if (bootverbose) {
1408 		kprintf("map %s intr %d on cpu%d\n",
1409 		    map->im_type == IOAPIC_IMT_MSI ? "MSI" : "MSI-X",
1410 		    intr, cpuid);
1411 	}
1412 
1413 	lwkt_reltoken(&ioapic_irqmap_tok);
1414 }
1415