1 #include "u.h"
2 #include "../port/lib.h"
3 #include "mem.h"
4 #include "dat.h"
5 #include "fns.h"
6
7 #include "apic.h"
8 #include "io.h"
9
10 typedef struct Rbus Rbus;
11 typedef struct Rdt Rdt;
12
13 struct Rbus {
14 Rbus *next;
15 int bustype;
16 int devno;
17 Rdt *rdt;
18 };
19
20 struct Rdt {
21 Apic* apic;
22 int intin;
23 u32int lo;
24
25 int ref; /* could map to multiple busses */
26 int enabled; /* times enabled */
27 };
28
29 enum { /* IOAPIC registers */
30 Ioregsel = 0x00, /* indirect register address */
31 Iowin = 0x04, /* indirect register data */
32 Ioipa = 0x08, /* IRQ Pin Assertion */
33 Ioeoi = 0x10, /* EOI */
34
35 Ioapicid = 0x00, /* Identification */
36 Ioapicver = 0x01, /* Version */
37 Ioapicarb = 0x02, /* Arbitration */
38 Ioabcfg = 0x03, /* Boot Coniguration */
39 Ioredtbl = 0x10, /* Redirection Table */
40 };
41
42 static Rdt rdtarray[Nrdt];
43 static int nrdtarray;
44 static int gsib;
45 static Rbus* rdtbus[Nbus];
46 static Rdt* rdtvecno[IdtMAX+1];
47
48 static Lock idtnolock;
49 static int idtno = IdtIOAPIC;
50
51 static void
rtblget(Apic * apic,int sel,u32int * hi,u32int * lo)52 rtblget(Apic* apic, int sel, u32int* hi, u32int* lo)
53 {
54 u32int r;
55
56 sel = Ioredtbl + 2*sel;
57
58 *(apic->addr+Ioregsel) = sel+1;
59 r = *(apic->addr+Iowin);
60 if(hi)
61 *hi = r;
62 *(apic->addr+Ioregsel) = sel;
63 r = *(apic->addr+Iowin);
64 if(lo)
65 *lo = r;
66 }
67
68 static void
rtblput(Apic * apic,int sel,u32int hi,u32int lo)69 rtblput(Apic* apic, int sel, u32int hi, u32int lo)
70 {
71 sel = Ioredtbl + 2*sel;
72
73 *(apic->addr+Ioregsel) = sel+1;
74 *(apic->addr+Iowin) = hi;
75 *(apic->addr+Ioregsel) = sel;
76 *(apic->addr+Iowin) = lo;
77 }
78
79 Rdt*
rdtlookup(Apic * apic,int intin)80 rdtlookup(Apic *apic, int intin)
81 {
82 int i;
83 Rdt *r;
84
85 for(i = 0; i < nrdtarray; i++){
86 r = rdtarray + i;
87 if(apic == r->apic && intin == r->intin)
88 return r;
89 }
90 return nil;
91 }
92
93 void
ioapicintrinit(int bustype,int busno,int apicno,int intin,int devno,u32int lo)94 ioapicintrinit(int bustype, int busno, int apicno, int intin, int devno, u32int lo)
95 {
96 Rbus *rbus;
97 Rdt *rdt;
98 Apic *apic;
99
100 if(busno >= Nbus || apicno >= Napic || nrdtarray >= Nrdt)
101 return;
102 apic = &ioapic[apicno];
103 if(!apic->useable || intin >= apic->nrdt)
104 return;
105
106 rdt = rdtlookup(apic, intin);
107 if(rdt == nil){
108 if(nrdtarray == nelem(rdtarray)){
109 print("ioapic: intrinit: rdtarray too small\n");
110 return;
111 }
112 rdt = &rdtarray[nrdtarray++];
113 rdt->apic = apic;
114 rdt->intin = intin;
115 rdt->lo = lo;
116 }else{
117 if(lo != rdt->lo){
118 if(bustype == BusISA && intin < 16 && lo == (Im|IPhigh|TMedge)){
119 DBG("override: isa %d %.8ux\n", intin, rdt->lo);
120 return; /* expected; default was overridden*/
121 }
122 print("multiple irq botch type %d bus %d %d/%d/%d lo %.8ux vs %.8ux\n",
123 bustype, busno, apicno, intin, devno, lo, rdt->lo);
124 return;
125 }
126 DBG("dup rdt %d %d %d %d %.8ux\n", busno, apicno, intin, devno, lo);
127 }
128 rdt->ref++;
129 rbus = malloc(sizeof(*rbus));
130 rbus->rdt = rdt;
131 rbus->bustype = bustype;
132 rbus->devno = devno;
133 rbus->next = rdtbus[busno];
134 rdtbus[busno] = rbus;
135 }
136
137 void
ioapicinit(int id,uintmem pa)138 ioapicinit(int id, uintmem pa)
139 {
140 Apic *apic;
141
142 /*
143 * Mark the IOAPIC useable if it has a good ID
144 * and the registers can be mapped.
145 */
146 if(id >= Napic)
147 return;
148
149 apic = &ioapic[id];
150 if(apic->useable || (apic->addr = vmap(pa, 1024)) == nil)
151 return;
152 apic->useable = 1;
153
154 /*
155 * Initialise the IOAPIC.
156 * The MultiProcessor Specification says it is the
157 * responsibility of the O/S to set the APIC ID.
158 */
159 lock(apic);
160 *(apic->addr+Ioregsel) = Ioapicver;
161 apic->nrdt = ((*(apic->addr+Iowin)>>16) & 0xff) + 1;
162 apic->gsib = gsib;
163 gsib += apic->nrdt;
164
165 *(apic->addr+Ioregsel) = Ioapicid;
166 *(apic->addr+Iowin) = id<<24;
167 unlock(apic);
168 }
169
170 void
ioapicdump(void)171 ioapicdump(void)
172 {
173 int i, n;
174 Rbus *rbus;
175 Rdt *rdt;
176 Apic *apic;
177 u32int hi, lo;
178
179 if(!DBGFLG)
180 return;
181
182 for(i = 0; i < Napic; i++){
183 apic = &ioapic[i];
184 if(!apic->useable || apic->addr == 0)
185 continue;
186 DBG("ioapic %d addr %#p nrdt %d gsib %d\n",
187 i, apic->addr, apic->nrdt, apic->gsib);
188 for(n = 0; n < apic->nrdt; n++){
189 lock(apic);
190 rtblget(apic, n, &hi, &lo);
191 unlock(apic);
192 DBG(" rdt %2.2d %#8.8ux %#8.8ux\n", n, hi, lo);
193 }
194 }
195 for(i = 0; i < Nbus; i++){
196 if((rbus = rdtbus[i]) == nil)
197 continue;
198 DBG("iointr bus %d:\n", i);
199 while(rbus != nil){
200 rdt = rbus->rdt;
201 DBG(" apic %ld devno %#ux (%d %d) intin %d lo %#ux ref %d\n",
202 rdt->apic-ioapic, rbus->devno, rbus->devno>>2,
203 rbus->devno & 0x03, rdt->intin, rdt->lo, rdt->ref);
204 rbus = rbus->next;
205 }
206 }
207 }
208
209 void
ioapiconline(void)210 ioapiconline(void)
211 {
212 int i;
213 Apic *apic;
214
215 for(apic = ioapic; apic < &ioapic[Napic]; apic++){
216 if(!apic->useable || apic->addr == nil)
217 continue;
218 for(i = 0; i < apic->nrdt; i++){
219 lock(apic);
220 rtblput(apic, i, 0, Im);
221 unlock(apic);
222 }
223 }
224 if(DBGFLG)
225 ioapicdump();
226 }
227
228 static int dfpolicy = 0;
229
230 static void
ioapicintrdd(u32int * hi,u32int * lo)231 ioapicintrdd(u32int* hi, u32int* lo)
232 {
233 int i;
234 static int df;
235 static Lock dflock;
236
237 /*
238 * Set delivery mode (lo) and destination field (hi),
239 * according to interrupt routing policy.
240 */
241 /*
242 * The bulk of this code was written ~1995, when there was
243 * one architecture and one generation of hardware, the number
244 * of CPUs was up to 4(8) and the choices for interrupt routing
245 * were physical, or flat logical (optionally with lowest
246 * priority interrupt). Logical mode hasn't scaled well with
247 * the increasing number of packages/cores/threads, so the
248 * fall-back is to physical mode, which works across all processor
249 * generations, both AMD and Intel, using the APIC and xAPIC.
250 *
251 * Interrupt routing policy can be set here.
252 */
253 switch(dfpolicy){
254 default: /* noise core 0 */
255 *hi = sys->machptr[0]->apicno<<24;
256 break;
257 case 1: /* round-robin */
258 /*
259 * Assign each interrupt to a different CPU on a round-robin
260 * Some idea of the packages/cores/thread topology would be
261 * useful here, e.g. to not assign interrupts to more than one
262 * thread in a core. But, as usual, Intel make that an onerous
263 * task.
264 */
265 lock(&dflock);
266 for(;;){
267 i = df++;
268 if(df >= MACHMAX+1)
269 df = 0;
270 if(sys->machptr[i] == nil || !sys->machptr[i]->online)
271 continue;
272 i = sys->machptr[i]->apicno;
273 if(xapic[i].useable && xapic[i].addr == 0)
274 break;
275 }
276 unlock(&dflock);
277
278 *hi = i<<24;
279 break;
280 }
281 *lo |= Pm|MTf;
282 }
283
284 int
ioapicintrenable(Vctl * v)285 ioapicintrenable(Vctl* v)
286 {
287 Rbus *rbus;
288 Rdt *rdt;
289 u32int hi, lo;
290 int bustype, busno, devno, vecno;
291
292 /*
293 * Bridge between old and unspecified new scheme,
294 * the work in progress...
295 */
296 if(v->tbdf == BUSUNKNOWN){
297 if(v->irq >= IdtLINT0 && v->irq <= IdtSPURIOUS){
298 if(v->irq != IdtSPURIOUS)
299 v->isr = apiceoi;
300 v->type = "lapic";
301 return v->irq;
302 }
303 else{
304 /*
305 * Legacy ISA.
306 * Make a busno and devno using the
307 * ISA bus number and the irq.
308 */
309 extern int mpisabusno;
310
311 if(mpisabusno == -1)
312 panic("no ISA bus allocated");
313 busno = mpisabusno;
314 devno = v->irq;
315 bustype = BusISA;
316 }
317 }
318 else if((bustype = BUSTYPE(v->tbdf)) == BusPCI){
319 /*
320 * PCI.
321 * Make a devno from BUSDNO(tbdf) and pcidev->intp.
322 */
323 Pcidev *pcidev;
324
325 busno = BUSBNO(v->tbdf);
326 if((pcidev = pcimatchtbdf(v->tbdf)) == nil)
327 panic("no PCI dev for tbdf %#8.8ux\n", v->tbdf);
328 if((devno = pcicfgr8(pcidev, PciINTP)) == 0)
329 panic("no INTP for tbdf %#8.8ux\n", v->tbdf);
330 devno = BUSDNO(v->tbdf)<<2|(devno-1);
331 DBG("ioapicintrenable: tbdf %#8.8ux busno %d devno %d\n",
332 v->tbdf, busno, devno);
333 }
334 else{
335 SET(busno, devno);
336 panic("unknown tbdf %#8.8ux\n", v->tbdf);
337 }
338
339 rdt = nil;
340 for(rbus = rdtbus[busno]; rbus != nil; rbus = rbus->next)
341 if(rbus->devno == devno && rbus->bustype == bustype){
342 rdt = rbus->rdt;
343 break;
344 }
345 if(rdt == nil){
346 extern int mpisabusno;
347
348 /*
349 * First crack in the smooth exterior of the new code:
350 * some BIOS make an MPS table where the PCI devices are
351 * just defaulted to ISA.
352 * Rewrite this to be cleaner.
353 */
354 if((busno = mpisabusno) == -1)
355 return -1;
356 devno = v->irq<<2;
357 for(rbus = rdtbus[busno]; rbus != nil; rbus = rbus->next){
358 if(rbus->devno == devno){
359 rdt = rbus->rdt;
360 break;
361 }
362 }
363 DBG("isa: tbdf %#8.8ux busno %d devno %d %#p\n",
364 v->tbdf, busno, devno, rdt);
365 }
366 if(rdt == nil)
367 return -1;
368
369 /*
370 * Second crack:
371 * what to do about devices that intrenable/intrdisable frequently?
372 * 1) there is no ioapicdisable yet;
373 * 2) it would be good to reuse freed vectors.
374 * Oh bugger.
375 */
376 /*
377 * This is a low-frequency event so just lock
378 * the whole IOAPIC to initialise the RDT entry
379 * rather than putting a Lock in each entry.
380 */
381 lock(rdt->apic);
382 if((rdt->lo & 0xff) == 0){
383 lock(&idtnolock);
384 vecno = idtno;
385 idtno = (idtno+8) % IdtMAX;
386 if(idtno < IdtIOAPIC)
387 idtno += IdtIOAPIC;
388 unlock(&idtnolock);
389
390 rdt->lo |= vecno;
391 rdtvecno[vecno] = rdt;
392 }
393
394 rdt->enabled++;
395 lo = (rdt->lo & ~Im);
396 ioapicintrdd(&hi, &lo);
397 rtblput(rdt->apic, rdt->intin, hi, lo);
398 vecno = lo & 0xff;
399 unlock(rdt->apic);
400
401 DBG("busno %d devno %d hi %#8.8ux lo %#8.8ux vecno %d\n",
402 busno, devno, hi, lo, vecno);
403 v->isr = apicisr;
404 v->eoi = apiceoi;
405 v->vno = vecno;
406 v->type = "ioapic";
407
408 return vecno;
409 }
410
411 int
ioapicintrdisable(int vecno)412 ioapicintrdisable(int vecno)
413 {
414 Rdt *rdt;
415
416 /*
417 * FOV. Oh dear. This isn't very good.
418 * Fortunately rdtvecno[vecno] is static
419 * once assigned.
420 * Must do better.
421 *
422 * What about any pending interrupts?
423 */
424 if(vecno < 0 || vecno > IdtMAX){
425 panic("ioapicintrdisable: vecno %d out of range", vecno);
426 return -1;
427 }
428 if((rdt = rdtvecno[vecno]) == nil){
429 panic("ioapicintrdisable: vecno %d has no rdt", vecno);
430 return -1;
431 }
432
433 lock(rdt->apic);
434 rdt->enabled--;
435 if(rdt->enabled == 0)
436 rtblput(rdt->apic, rdt->intin, 0, rdt->lo);
437 unlock(rdt->apic);
438
439 return 0;
440 }
441