1 #include "u.h"
2 #include "../port/lib.h"
3 #include "mem.h"
4 #include "dat.h"
5 #include "fns.h"
6 #include "ureg.h"
7
8 #define TLBINVAL(x, pid) puttlbx(x, KZERO|((x)<<(PGSHIFT+1))|(pid), 0, 0, PGSZ)
9
10 enum {
11 Debugswitch = 0,
12 Debughash = 0,
13 };
14
15 static ulong ktime[8]; /* only for first 8 cpus */
16
17 void
tlbinit(void)18 tlbinit(void)
19 {
20 int i;
21
22 for(i=0; i<NTLB; i++)
23 TLBINVAL(i, 0);
24 }
25
26 Lock kmaplock;
27 KMap kpte[KPTESIZE];
28 KMap* kmapfree;
29
30 static int minfree = KPTESIZE;
31 static int lastfree;
32 static int tlbroff = TLBROFF;
33
34 void
kmapinit(void)35 kmapinit(void)
36 {
37 KMap *k, *klast;
38
39 lock(&kmaplock);
40 kmapfree = kpte;
41 klast = &kpte[KPTESIZE-1];
42 for(k=kpte; k<klast; k++)
43 k->next = k+1;
44 k->next = 0;
45 unlock(&kmaplock);
46
47 m->ktlbnext = KTLBOFF;
48 }
49
50 void
kmapinval(void)51 kmapinval(void)
52 {
53 int mno, i, curpid;
54 KMap *k, *next;
55 uchar *ktlbx;
56
57 if(m->machno < nelem(ktime))
58 ktime[m->machno] = MACHP(0)->ticks;
59 if(m->kactive == 0)
60 return;
61
62 curpid = PTEPID(TLBPID(tlbvirt()));
63 ktlbx = m->ktlbx;
64 for(i = 0; i < NTLB; i++, ktlbx++){
65 if(*ktlbx == 0)
66 continue;
67 TLBINVAL(i, curpid);
68 *ktlbx = 0;
69 }
70
71 mno = m->machno;
72 for(k = m->kactive; k; k = next) {
73 next = k->konmach[mno];
74 kunmap(k);
75 }
76
77 m->kactive = 0;
78 m->ktlbnext = KTLBOFF;
79 }
80
81 static int
putktlb(KMap * k)82 putktlb(KMap *k)
83 {
84 int x;
85 ulong virt;
86 ulong tlbent[3];
87
88 virt = k->virt & ~BY2PG | TLBPID(tlbvirt());
89 x = gettlbp(virt, tlbent);
90 if (!m->paststartup)
91 if (up) { /* startup just ended? */
92 tlbroff = KTLBOFF;
93 setwired(tlbroff);
94 m->paststartup = 1;
95 } else if (x < 0) { /* no such entry? use next */
96 x = m->ktlbnext++;
97 if(m->ktlbnext >= tlbroff)
98 m->ktlbnext = KTLBOFF;
99 }
100 if (x < 0) /* no entry for va? overwrite random one */
101 x = puttlb(virt, k->phys0, k->phys1);
102 else
103 puttlbx(x, virt, k->phys0, k->phys1, PGSZ);
104 m->ktlbx[x] = 1;
105 return x;
106 }
107
108 /*
109 * Arrange that the KMap'd virtual address will hit the same
110 * primary cache line as pg->va by making bits 14...12 of the
111 * tag the same as virtual address. These bits are the index
112 * into the primary cache and are checked whenever accessing
113 * the secondary cache through the primary. Violation causes
114 * a VCE trap.
115 */
116 KMap *
kmap(Page * pg)117 kmap(Page *pg)
118 {
119 int s, printed = 0;
120 ulong pte, virt;
121 KMap *k;
122
123 s = splhi();
124 lock(&kmaplock);
125
126 if(kmapfree == 0) {
127 retry:
128 unlock(&kmaplock);
129 kmapinval(); /* try and free some */
130 lock(&kmaplock);
131 if(kmapfree == 0){
132 unlock(&kmaplock);
133 splx(s);
134 if(printed++ == 0){
135 /* using iprint here we get mixed up with other prints */
136 print("%d KMAP RETRY %#lux ktime %ld %ld %ld %ld %ld %ld %ld %ld\n",
137 m->machno, getcallerpc(&pg),
138 ktime[0], ktime[1], ktime[2], ktime[3],
139 ktime[4], ktime[5], ktime[6], ktime[7]);
140 delay(200);
141 }
142 splhi();
143 lock(&kmaplock);
144 goto retry;
145 }
146 }
147
148 k = kmapfree;
149 kmapfree = k->next;
150
151 k->pg = pg;
152 /*
153 * One for the allocation,
154 * One for kactive
155 */
156 k->pc = getcallerpc(&pg);
157 k->ref = 2;
158 k->konmach[m->machno] = m->kactive;
159 m->kactive = k;
160
161 virt = pg->va;
162 /* bits 13..12 form the cache virtual index */
163 virt &= PIDX;
164 virt |= KMAPADDR | ((k-kpte)<<KMAPSHIFT);
165
166 k->virt = virt;
167 pte = PPN(pg->pa)|PTECACHABILITY|PTEGLOBL|PTEWRITE|PTEVALID;
168 if(virt & BY2PG) {
169 k->phys0 = PTEGLOBL | PTECACHABILITY;
170 k->phys1 = pte;
171 }
172 else {
173 k->phys0 = pte;
174 k->phys1 = PTEGLOBL | PTECACHABILITY;
175 }
176
177 putktlb(k);
178 unlock(&kmaplock);
179
180 splx(s);
181 return k;
182 }
183
184 void
kunmap(KMap * k)185 kunmap(KMap *k)
186 {
187 int s;
188
189 s = splhi();
190 if(decref(k) == 0) {
191 k->virt = 0;
192 k->phys0 = 0;
193 k->phys1 = 0;
194 k->pg = 0;
195
196 lock(&kmaplock);
197 k->next = kmapfree;
198 kmapfree = k;
199 //nfree();
200 unlock(&kmaplock);
201 }
202 splx(s);
203 }
204
205 void
kfault(Ureg * ur)206 kfault(Ureg *ur) /* called from trap() */
207 {
208 ulong index, addr;
209 KMap *k, *f;
210
211 addr = ur->badvaddr;
212 index = (addr & ~KSEGM) >> KMAPSHIFT;
213 if(index >= KPTESIZE)
214 panic("kmapfault: %#lux", addr);
215
216 k = &kpte[index];
217 if(k->virt == 0)
218 panic("kmapfault: unmapped %#lux", addr);
219
220 for(f = m->kactive; f; f = f->konmach[m->machno])
221 if(f == k)
222 break;
223 if(f == 0) {
224 incref(k);
225 k->konmach[m->machno] = m->kactive;
226 m->kactive = k;
227 }
228 putktlb(k);
229 }
230
231 struct
232 {
233 ulong va;
234 ulong pl;
235 ulong ph;
236 } wired[NWTLB+1]; /* +1 to avoid zero size if NWTLB==0 */
237
238 void
machwire(void)239 machwire(void)
240 {
241 int i;
242
243 //if(m->machno == 0)
244 // return;
245 for(i = 0; i < NWTLB; i++)
246 if(wired[i].va)
247 puttlbx(i+WTLBOFF, wired[i].va, wired[i].pl,
248 wired[i].ph, PGSZ);
249 }
250
251 /*
252 * Process must be splhi
253 */
254 int
newtlbpid(Proc * p)255 newtlbpid(Proc *p)
256 {
257 int i, s;
258 Proc **h;
259
260 i = m->lastpid;
261 h = m->pidproc;
262 for(s = 0; s < NTLBPID; s++) {
263 i++;
264 if(i >= NTLBPID)
265 i = 1;
266 if(h[i] == 0)
267 break;
268 }
269
270 if(h[i])
271 purgetlb(i);
272 if(h[i] != 0)
273 panic("newtlb");
274
275 m->pidproc[i] = p;
276 p->pidonmach[m->machno] = i;
277 m->lastpid = i;
278
279 return i;
280 }
281
282 void
mmuswitch(Proc * p)283 mmuswitch(Proc *p)
284 {
285 int tp;
286 static char lasttext[32];
287
288 if(Debugswitch && !p->kp){
289 if(strncmp(lasttext, p->text, sizeof lasttext) != 0)
290 iprint("[%s]", p->text);
291 strncpy(lasttext, p->text, sizeof lasttext);
292 }
293
294 if(p->newtlb) {
295 memset(p->pidonmach, 0, sizeof p->pidonmach);
296 p->newtlb = 0;
297 }
298 tp = p->pidonmach[m->machno];
299 if(tp == 0)
300 tp = newtlbpid(p);
301 puttlbx(0, KZERO|PTEPID(tp), 0, 0, PGSZ);
302 }
303
304 void
mmurelease(Proc * p)305 mmurelease(Proc *p)
306 {
307 memset(p->pidonmach, 0, sizeof p->pidonmach);
308 }
309
310 void
putmmu(ulong tlbvirt,ulong tlbphys,Page * pg)311 putmmu(ulong tlbvirt, ulong tlbphys, Page *pg)
312 {
313 short tp;
314 char *ctl;
315 Softtlb *entry;
316 int s;
317
318 s = splhi();
319 tp = up->pidonmach[m->machno];
320 if(tp == 0)
321 tp = newtlbpid(up);
322
323 tlbvirt |= PTEPID(tp);
324 if((tlbphys & PTEALGMASK) != PTEUNCACHED) {
325 tlbphys &= ~PTEALGMASK;
326 tlbphys |= PTECACHABILITY;
327 }
328
329 entry = putstlb(tlbvirt, tlbphys);
330 puttlb(entry->virt, entry->phys0, entry->phys1);
331
332 ctl = &pg->cachectl[m->machno];
333 switch(*ctl) {
334 case PG_TXTFLUSH:
335 icflush((void*)pg->va, BY2PG);
336 *ctl = PG_NOFLUSH;
337 break;
338 case PG_DATFLUSH:
339 dcflush((void*)pg->va, BY2PG);
340 *ctl = PG_NOFLUSH;
341 break;
342 case PG_NEWCOL:
343 cleancache(); /* Too expensive */
344 *ctl = PG_NOFLUSH;
345 break;
346 }
347 splx(s);
348 }
349
350 void
purgetlb(int pid)351 purgetlb(int pid)
352 {
353 int i, mno;
354 Proc *sp, **pidproc;
355 Softtlb *entry, *etab;
356
357 m->tlbpurge++;
358
359 /*
360 * find all pid entries that are no longer used by processes
361 */
362 mno = m->machno;
363 pidproc = m->pidproc;
364 for(i=1; i<NTLBPID; i++) {
365 sp = pidproc[i];
366 if(sp && sp->pidonmach[mno] != i)
367 pidproc[i] = 0;
368 }
369
370 /*
371 * shoot down the one we want
372 */
373 sp = pidproc[pid];
374 if(sp != 0)
375 sp->pidonmach[mno] = 0;
376 pidproc[pid] = 0;
377
378 /*
379 * clean out all dead pids from the stlb;
380 */
381 entry = m->stb;
382 for(etab = &entry[STLBSIZE]; entry < etab; entry++)
383 if(pidproc[TLBPID(entry->virt)] == 0)
384 entry->virt = 0;
385
386 /*
387 * clean up the hardware
388 */
389 for(i=tlbroff; i<NTLB; i++)
390 if(pidproc[TLBPID(gettlbvirt(i))] == 0)
391 TLBINVAL(i, pid);
392 }
393
394 void
flushmmu(void)395 flushmmu(void)
396 {
397 int s;
398
399 s = splhi();
400 up->newtlb = 1;
401 mmuswitch(up);
402 splx(s);
403 }
404
405 /* tlbvirt also has TLBPID() in its low byte as the asid */
406 Softtlb*
putstlb(ulong tlbvirt,ulong tlbphys)407 putstlb(ulong tlbvirt, ulong tlbphys)
408 {
409 int odd;
410 Softtlb *entry;
411
412 /* identical calculation in l.s/utlbmiss */
413 entry = &m->stb[stlbhash(tlbvirt)];
414 odd = tlbvirt & BY2PG; /* even/odd bit */
415 tlbvirt &= ~BY2PG; /* zero even/odd bit */
416 if(entry->virt != tlbvirt) { /* not my entry? overwrite it */
417 if(entry->virt != 0) {
418 m->hashcoll++;
419 if (Debughash)
420 iprint("putstlb: hash collision: %#lx old virt "
421 "%#lux new virt %#lux page %#lux\n",
422 entry - m->stb, entry->virt, tlbvirt,
423 tlbvirt >> (PGSHIFT+1));
424 }
425 entry->virt = tlbvirt;
426 entry->phys0 = 0;
427 entry->phys1 = 0;
428 }
429
430 if(odd)
431 entry->phys1 = tlbphys;
432 else
433 entry->phys0 = tlbphys;
434
435 if(entry->phys0 == 0 && entry->phys1 == 0)
436 entry->virt = 0;
437
438 return entry;
439 }
440
441 void
checkmmu(ulong,ulong)442 checkmmu(ulong, ulong)
443 {
444 }
445
446 void
countpagerefs(ulong *,int)447 countpagerefs(ulong*, int)
448 {
449 }
450
451 /*
452 * Return the number of bytes that can be accessed via KADDR(pa).
453 * If pa is not a valid argument to KADDR, return 0.
454 */
455 uintptr
cankaddr(uintptr pa)456 cankaddr(uintptr pa)
457 {
458 if(pa >= KZERO || pa >= memsize)
459 return 0;
460 return memsize - pa;
461 }
462
463 /* print tlb entries for debug */
464 #define TLBPHYS(x) ((((x)&~0x3f)<<6) | ((x)&0x3f)) /* phys addr & flags */
465
466 void
dumptlb(void)467 dumptlb(void)
468 {
469 Softtlb entry;
470 int i;
471
472 iprint("dump tlb\n");
473 for(i=0; i<NTLB; i++) {
474 gettlbx(i, &entry);
475 if(entry.phys0 != 0 || entry.phys1 != 0)
476 iprint("tlb index %2d, virt %.8lux, phys0 %.8lux, phys1 %.8lux\n",
477 i, entry.virt, TLBPHYS(entry.phys0), TLBPHYS(entry.phys1));
478 }
479 }
480
481 void
dumpstlb(void)482 dumpstlb(void)
483 {
484 Softtlb *entry;
485 int i;
486
487 iprint("dump stlb\n");
488 for(i=0; i<STLBSIZE; i++) {
489 entry = &m->stb[i];
490 if(entry->virt != 0)
491 iprint("stlb index %2d, virt %.8lux, phys0 %.8lux, phys1 %.8lux\n",
492 i, entry->virt, TLBPHYS(entry->phys0), TLBPHYS(entry->phys1));
493 }
494 }
495