1 #include "u.h"
2 #include "../port/lib.h"
3 #include "mem.h"
4 #include "dat.h"
5 #include "fns.h"
6 #include "ureg.h"
7
8 #define TLBINVAL(x, pid) puttlbx(x, KSEG0|((x)<<(PGSHIFT+1))|(pid), 0, 0, PGSZ)
9
10 enum {
11 Debugswitch = 0,
12 Debughash = 0,
13 };
14
15 void dumpstlb(void);
16
17 static ulong ktime[8]; /* only for first 8 cpus */
18
19 void
tlbinit(void)20 tlbinit(void)
21 {
22 int i;
23
24 for(i=0; i<NTLB; i++)
25 TLBINVAL(i, 0);
26 }
27
28 Lock kmaplock;
29 KMap kpte[KPTESIZE];
30 KMap* kmapfree;
31
32 static int minfree = KPTESIZE;
33 static int lastfree;
34 static int tlbroff = TLBROFF;
35
36 void
kmapinit(void)37 kmapinit(void)
38 {
39 KMap *k, *klast;
40
41 lock(&kmaplock);
42 kmapfree = kpte;
43 klast = &kpte[KPTESIZE-1];
44 for(k=kpte; k<klast; k++)
45 k->next = k+1;
46 k->next = 0;
47 unlock(&kmaplock);
48
49 m->ktlbnext = KTLBOFF;
50 }
51
52 void
kmapinval(void)53 kmapinval(void)
54 {
55 int mno, i, curpid;
56 KMap *k, *next;
57 uchar *ktlbx;
58
59 if(m->machno < nelem(ktime))
60 ktime[m->machno] = MACHP(0)->ticks;
61 if(m->kactive == 0)
62 return;
63
64 curpid = PTEPID(TLBPID(tlbvirt()));
65 ktlbx = m->ktlbx;
66 for(i = 0; i < NTLB; i++, ktlbx++){
67 if(*ktlbx == 0)
68 continue;
69 TLBINVAL(i, curpid);
70 *ktlbx = 0;
71 }
72
73 mno = m->machno;
74 for(k = m->kactive; k; k = next) {
75 next = k->konmach[mno];
76 kunmap(k);
77 }
78
79 m->kactive = 0;
80 m->ktlbnext = KTLBOFF;
81 }
82
83 static int
putktlb(KMap * k)84 putktlb(KMap *k)
85 {
86 int x;
87 u64int virt, tlbent[4];
88
89 virt = k->virt & ~BY2PG | TLBPID(tlbvirt());
90 x = gettlbp(virt, tlbent);
91 if (!m->paststartup)
92 if (up) { /* startup just ended? */
93 tlbroff = KTLBOFF;
94 setwired(tlbroff);
95 m->paststartup = 1;
96 } else if (x < 0) { /* no such entry? use next */
97 x = m->ktlbnext++;
98 if(m->ktlbnext >= tlbroff)
99 m->ktlbnext = KTLBOFF;
100 }
101 if (x < 0) /* no entry for va? overwrite random one */
102 x = puttlb(virt, k->phys0, k->phys1);
103 else
104 puttlbx(x, virt, k->phys0, k->phys1, PGSZ);
105 m->ktlbx[x] = 1;
106 return x;
107 }
108
109 /*
110 * Arrange that the KMap'd virtual address will hit the same
111 * primary cache line as pg->va by making bits 14...12 of the
112 * tag the same as virtual address. These bits are the index
113 * into the primary cache and are checked whenever accessing
114 * the secondary cache through the primary. Violation causes
115 * a VCE trap.
116 */
117 KMap *
kmap(Page * pg)118 kmap(Page *pg)
119 {
120 int s, printed = 0;
121 uintptr pte, virt;
122 KMap *k;
123
124 s = splhi();
125 lock(&kmaplock);
126
127 if(kmapfree == 0) {
128 retry:
129 unlock(&kmaplock);
130 kmapinval(); /* try and free some */
131 lock(&kmaplock);
132 if(kmapfree == 0){
133 unlock(&kmaplock);
134 splx(s);
135 if(printed++ == 0){
136 /* using iprint here we get mixed up with other prints */
137 print("%d KMAP RETRY %#p ktime %ld %ld %ld %ld %ld %ld %ld %ld\n",
138 m->machno, getcallerpc(&pg),
139 ktime[0], ktime[1], ktime[2], ktime[3],
140 ktime[4], ktime[5], ktime[6], ktime[7]);
141 delay(200);
142 }
143 splhi();
144 lock(&kmaplock);
145 goto retry;
146 }
147 }
148
149 k = kmapfree;
150 kmapfree = k->next;
151
152 k->pg = pg;
153 /*
154 * One for the allocation,
155 * One for kactive
156 */
157 k->pc = getcallerpc(&pg);
158 k->ref = 2;
159 k->konmach[m->machno] = m->kactive;
160 m->kactive = k;
161
162 virt = pg->va;
163 /* bits 13..12 form the cache virtual index */
164 virt &= PIDX;
165 virt |= KMAPADDR | ((k-kpte)<<KMAPSHIFT);
166
167 k->virt = virt;
168 pte = PPN(pg->pa)|PTECACHABILITY|PTEGLOBL|PTEWRITE|PTEVALID;
169 if(virt & BY2PG) {
170 k->phys0 = PTEGLOBL | PTECACHABILITY;
171 k->phys1 = pte;
172 }
173 else {
174 k->phys0 = pte;
175 k->phys1 = PTEGLOBL | PTECACHABILITY;
176 }
177
178 putktlb(k);
179 unlock(&kmaplock);
180
181 splx(s);
182 return k;
183 }
184
185 void
kunmap(KMap * k)186 kunmap(KMap *k)
187 {
188 int s;
189
190 s = splhi();
191 if(decref(k) == 0) {
192 k->virt = 0;
193 k->phys0 = 0;
194 k->phys1 = 0;
195 k->pg = 0;
196
197 lock(&kmaplock);
198 k->next = kmapfree;
199 kmapfree = k;
200 //nfree();
201 unlock(&kmaplock);
202 }
203 splx(s);
204 }
205
206 void
kfault(Ureg * ur)207 kfault(Ureg *ur) /* called from trap() */
208 {
209 ulong index;
210 uintptr addr;
211 KMap *k, *f;
212
213 addr = ur->badvaddr;
214 index = (addr & ~KSEGM) >> KMAPSHIFT;
215 if(index >= KPTESIZE) {
216 dumpregs(ur);
217 dumptlb();
218 dumpstlb();
219 panic("kmapfault: %#p", addr);
220 }
221
222 k = &kpte[index];
223 if(k->virt == 0) {
224 dumptlb();
225 dumpstlb();
226 panic("kmapfault: unmapped %#p", addr);
227 }
228
229 for(f = m->kactive; f; f = f->konmach[m->machno])
230 if(f == k)
231 break;
232 if(f == 0) {
233 incref(k);
234 k->konmach[m->machno] = m->kactive;
235 m->kactive = k;
236 }
237 putktlb(k);
238 }
239
240 struct
241 {
242 u64int va;
243 u64int pl;
244 u64int ph;
245 } wired[NWTLB+1]; /* +1 to avoid zero size if NWTLB==0 */
246
247 void
machwire(void)248 machwire(void)
249 {
250 int i;
251
252 // if(m->machno == 0)
253 // return;
254 for(i = 0; i < NWTLB; i++)
255 if(wired[i].va)
256 puttlbx(i+WTLBOFF, wired[i].va, wired[i].pl,
257 wired[i].ph, PGSZ);
258 }
259
260 /*
261 * Process must be splhi
262 */
263 int
newtlbpid(Proc * p)264 newtlbpid(Proc *p)
265 {
266 int i, s;
267 Proc **h;
268
269 i = m->lastpid;
270 h = m->pidproc;
271 for(s = 0; s < NTLBPID; s++) {
272 i++;
273 if(i >= NTLBPID)
274 i = 1;
275 if(h[i] == 0)
276 break;
277 }
278
279 if(h[i])
280 purgetlb(i);
281 if(h[i] != 0)
282 panic("newtlb");
283
284 m->pidproc[i] = p;
285 p->pidonmach[m->machno] = i;
286 m->lastpid = i;
287
288 return i;
289 }
290
291 void
mmuswitch(Proc * p)292 mmuswitch(Proc *p)
293 {
294 int tp;
295 static char lasttext[32];
296
297 if(Debugswitch && !p->kp){
298 if(strncmp(lasttext, p->text, sizeof lasttext) != 0)
299 iprint("[%s]", p->text);
300 strncpy(lasttext, p->text, sizeof lasttext);
301 }
302
303 if(p->newtlb) {
304 memset(p->pidonmach, 0, sizeof p->pidonmach);
305 p->newtlb = 0;
306 }
307 tp = p->pidonmach[m->machno];
308 if(tp == 0)
309 tp = newtlbpid(p);
310 puttlbx(0, KSEG0|PTEPID(tp), 0, 0, PGSZ);
311 }
312
313 void
mmurelease(Proc * p)314 mmurelease(Proc *p)
315 {
316 memset(p->pidonmach, 0, sizeof p->pidonmach);
317 }
318
319 void
putmmu(u64int tlbvirt,u64int tlbphys,Page * pg)320 putmmu(u64int tlbvirt, u64int tlbphys, Page *pg)
321 {
322 short tp;
323 char *ctl;
324 Softtlb *entry;
325 int s;
326
327 s = splhi();
328 tp = up->pidonmach[m->machno];
329 if(tp == 0)
330 tp = newtlbpid(up);
331
332 tlbvirt |= PTEPID(tp);
333 if((tlbphys & PTEALGMASK) != PTEUNCACHED) {
334 tlbphys &= ~PTEALGMASK;
335 tlbphys |= PTECACHABILITY;
336 }
337
338 entry = putstlb(tlbvirt, tlbphys);
339 puttlb(entry->virt, entry->phys0, entry->phys1);
340
341 ctl = &pg->cachectl[m->machno];
342 switch(*ctl) {
343 case PG_TXTFLUSH:
344 icflush((void*)pg->va, BY2PG);
345 *ctl = PG_NOFLUSH;
346 break;
347 case PG_DATFLUSH:
348 dcflush((void*)pg->va, BY2PG);
349 *ctl = PG_NOFLUSH;
350 break;
351 case PG_NEWCOL:
352 cleancache(); /* Too expensive */
353 *ctl = PG_NOFLUSH;
354 break;
355 }
356 splx(s);
357 }
358
359 void
purgetlb(int pid)360 purgetlb(int pid)
361 {
362 int i, mno;
363 Proc *sp, **pidproc;
364 Softtlb *entry, *etab;
365
366 m->tlbpurge++;
367
368 /*
369 * find all pid entries that are no longer used by processes
370 */
371 mno = m->machno;
372 pidproc = m->pidproc;
373 for(i=1; i<NTLBPID; i++) {
374 sp = pidproc[i];
375 if(sp && sp->pidonmach[mno] != i)
376 pidproc[i] = 0;
377 }
378
379 /*
380 * shoot down the one we want
381 */
382 sp = pidproc[pid];
383 if(sp != 0)
384 sp->pidonmach[mno] = 0;
385 pidproc[pid] = 0;
386
387 /*
388 * clean out all dead pids from the stlb;
389 */
390 entry = m->stb;
391 for(etab = &entry[STLBSIZE]; entry < etab; entry++)
392 if(pidproc[TLBPID(entry->virt)] == 0)
393 entry->virt = 0;
394
395 /*
396 * clean up the hardware
397 */
398 for(i=tlbroff; i<NTLB; i++)
399 if(pidproc[TLBPID(gettlbvirt(i))] == 0)
400 TLBINVAL(i, pid);
401 }
402
403 void
flushmmu(void)404 flushmmu(void)
405 {
406 int s;
407
408 s = splhi();
409 up->newtlb = 1;
410 mmuswitch(up);
411 splx(s);
412 }
413
414 /* tlbvirt also has TLBPID() in its low byte as the asid */
415 Softtlb*
putstlb(u64int tlbvirt,u64int tlbphys)416 putstlb(u64int tlbvirt, u64int tlbphys)
417 {
418 int odd;
419 Softtlb *entry;
420
421 /* identical calculation in l.s/utlbmiss */
422 entry = &m->stb[stlbhash(tlbvirt)];
423 odd = tlbvirt & BY2PG; /* even/odd bit */
424 tlbvirt &= ~BY2PG; /* zero even/odd bit */
425 if(entry->virt != (tlbvirt & TLBVIRTMASK)) { /* not my entry? overwrite it */
426 if(entry->virt != 0) {
427 m->hashcoll++;
428 if (Debughash)
429 iprint("putstlb: hash collision: %#lx old virt "
430 "%#p new virt %#p page %#llux\n",
431 entry - m->stb, entry->virt, tlbvirt,
432 tlbvirt >> (PGSHIFT+1));
433 }
434 entry->virt = tlbvirt & TLBVIRTMASK;
435 entry->phys0 = 0;
436 entry->phys1 = 0;
437 }
438
439 if(odd)
440 entry->phys1 = tlbphys;
441 else
442 entry->phys0 = tlbphys;
443
444 if(entry->phys0 == 0 && entry->phys1 == 0)
445 entry->virt = 0;
446
447 return entry;
448 }
449
450 void
checkmmu(ulong,ulong)451 checkmmu(ulong, ulong)
452 {
453 }
454
455 void
countpagerefs(ulong *,int)456 countpagerefs(ulong*, int)
457 {
458 }
459
460 /*
461 * Return the number of bytes that can be accessed via KADDR(pa).
462 * If pa is not a valid argument to KADDR, return 0.
463 */
464 uintptr
cankaddr(uintptr pa)465 cankaddr(uintptr pa)
466 {
467 if(pa >= KZERO || pa >= memsize)
468 return 0;
469 return memsize - pa;
470 }
471
472 /* print tlb entries for debug */
473 #define TLBPHYS(x) ((((x)&~0x3f)<<6) | ((x)&0x3f)) /* phys addr & flags */
474
475 void
dumptlb(void)476 dumptlb(void)
477 {
478 Softtlb entry;
479 int i;
480
481 iprint("dump tlb\n");
482 for(i=0; i<NTLB; i++) {
483 gettlbx(i, &entry);
484 // if(entry.phys0 != 0 || entry.phys1 != 0)
485 iprint("tlb index %2d, virt %.16llux, phys0 %.16llux, phys1 %.16llux\n",
486 i, entry.virt, TLBPHYS(entry.phys0), TLBPHYS(entry.phys1));
487 }
488 }
489
490 void
dumpstlb(void)491 dumpstlb(void)
492 {
493 Softtlb *entry;
494 int i;
495
496 iprint("dump stlb\n");
497 for(i=0; i<STLBSIZE; i++) {
498 entry = &m->stb[i];
499 if(entry->virt != 0)
500 iprint("stlb index %2d, virt %.16llux, phys0 %.16llux, phys1 %.16llux\n",
501 i, entry->virt, TLBPHYS(entry->phys0), TLBPHYS(entry->phys1));
502 }
503 }
504