1 #include "u.h"
2 #include "../port/lib.h"
3 #include "mem.h"
4 #include "dat.h"
5 #include "fns.h"
6
7 /*
8 * 440 mmu
9 *
10 * l.s has set some global TLB entries for the kernel at
11 * the top of the tlb (NTLB-1 down), partly to account for
12 * the way the firmware sets things up on some platforms (eg, 440).
13 * The first entry not used by the kernel (eg, by kmapphys) is m->utlbhi.
14 * User tlbs are assigned indices from 0 to m->utlbhi, round-robin.
15 * m->utlbnext is the next to use. Kernel tlbs are added at utlbhi, which then moves.
16 *
17 * In this version, the kernel TLB entries are in zone 0,
18 * and user pages are in zone 1. The kernel entries are also PID 0 (global)
19 * so they are usable as the PID changes (0=no user; non-zero=user process mapped).
20 */
21
22 enum {
23 Minutlb= NTLB/2, /* at least this many for user process */
24 };
25
26 #define DEBUG 0
27
28 Softtlb softtlb[MAXMACH][STLBSIZE];
29
30 static int newtlbpid(Proc*);
31 static void purgetlb(int);
32 static void putstlb(int, u32int, u32int, u32int);
33
34 static int
lookstlb(int pid,u32int va)35 lookstlb(int pid, u32int va)
36 {
37 int i;
38
39 pid <<= 2;
40 i = ((va>>12)^pid)&(STLBSIZE-1);
41 if(m->stlb[i].hi == (va|pid))
42 return i;
43
44 return -1;
45 }
46
47 void
tlbdump(char * s)48 tlbdump(char *s)
49 {
50 int i, p;
51 u32int hi, lo, mi, stid;
52
53 p = pidget();
54 iprint("tlb[%s] pid %d lastpid %d utlbnext %d\n",
55 s, p, m->lastpid, m->utlbnext);
56 for(i=0; i<NTLB; i++){
57 hi = tlbrehi(i);
58 if(hi & TLBVALID){
59 stid = stidget();
60 lo = tlbrelo(i);
61 mi = tlbremd(i);
62 iprint("%.2d %8.8ux %8.8ux %8.8ux %3d %2d\n",
63 i, hi, mi, lo, stid, lookstlb(i, TLBEPN(hi)));
64 }
65 if(i == m->utlbhi)
66 iprint("-----\n");
67 }
68 stidput(p); /* tlbrehi changes STID */
69 }
70
71 void
alltlbdump(void)72 alltlbdump(void)
73 {
74 Mreg s;
75 Proc *p;
76 int i, machno;
77
78 s = splhi();
79 machno = m->machno;
80 iprint("cpu%d pidproc:\n", machno);
81 for(i=0; i<nelem(m->pidproc); i++){
82 p = m->pidproc[i];
83 if(p != nil && (p->mmupid != i || p == up))
84 iprint("%d -> %ld [%d]\n", i, p->pid, p->mmupid);
85 }
86 tlbdump("final");
87 splx(s);
88 }
89
90 /*
91 * l.s provides a set of tlb entries for the kernel, allocating
92 * them starting at the highest index down. user-level entries
93 * will be allocated from index 0 up.
94 */
95 void
mmuinit(void)96 mmuinit(void)
97 {
98 int i;
99
100 pidput(0);
101 stidput(0);
102 m->stlb = softtlb[m->machno];
103 if(DEBUG)
104 tlbdump("init0");
105
106 m->lastpid = 0;
107 m->utlbhi = 0;
108 for(i = 0; i < NTLB; i++){
109 if(tlbrehi(i) & TLBVALID)
110 break;
111 m->utlbhi = i;
112 stidput(0);
113 tlbwrx(i, 0, 0, 0);
114 }
115 pidput(0);
116 stidput(0);
117 if(DEBUG)
118 tlbdump("init1");
119 }
120
121 void
flushmmu(void)122 flushmmu(void)
123 {
124 Mreg s;
125
126 s = splhi();
127 up->newtlb = 1;
128 mmuswitch(up);
129 splx(s);
130 }
131
132 /*
133 * called with splhi
134 */
135 void
mmuswitch(Proc * p)136 mmuswitch(Proc *p)
137 {
138 int pid;
139
140 if(p->newtlb){
141 p->mmupid = 0;
142 p->newtlb = 0;
143 }
144 pid = p->mmupid;
145 if(pid == 0 && !p->kp)
146 pid = newtlbpid(p);
147 pidput(pid);
148 stidput(pid);
149 }
150
151 void
mmurelease(Proc * p)152 mmurelease(Proc* p)
153 {
154 p->mmupid = 0;
155 pidput(0);
156 stidput(0);
157 }
158
159 void
putmmu(uintptr va,uintptr pa,Page * page)160 putmmu(uintptr va, uintptr pa, Page* page)
161 {
162 Mreg s;
163 int x, tp;
164 char *ctl;
165 u32int tlbhi, tlblo, tlbmi;
166
167 if(va >= KZERO)
168 panic("mmuput");
169
170 tlbhi = TLBEPN(va) | TLB4K | TLBVALID;
171 tlbmi = TLBRPN(pa); /* TO DO: 36-bit physical memory */
172 tlblo = TLBUR | TLBUX | TLBSR; /* shouldn't need TLBSX */
173 if(pa & PTEWRITE)
174 tlblo |= TLBUW | TLBSW;
175 if(pa & PTEUNCACHED)
176 tlblo |= TLBI | TLBG;
177
178 s = splhi();
179 tp = up->mmupid;
180 if(tp == 0){
181 if(up->kp)
182 panic("mmuput kp");
183 tp = newtlbpid(up);
184 pidput(tp);
185 stidput(tp);
186 }
187 else if(pidget() != tp || stidget() != tp){
188 iprint("mmuput: cpu%d pid up->mmupid=%d pidget=%d stidget=%d s=%#ux\n",
189 m->machno, tp, pidget(), stidget(), s);
190 alltlbdump();
191 panic("mmuput: cpu%d pid %d %d s=%#ux",
192 m->machno, tp, pidget(), s);
193 }
194
195 /* see if it's already there: note that tlbsx[cc] uses STID */
196 x = tlbsxcc(va);
197 if(x < 0){
198 if(m->utlbnext > m->utlbhi)
199 m->utlbnext = 0;
200 x = m->utlbnext++;
201 }else if(x > m->utlbhi)
202 panic("mmuput index va=%#p x=%d klo=%d", va, x, m->utlbhi); /* shouldn't touch kernel entries */
203 // if(DEBUG)
204 // iprint("put %#p %#p-> %d: %8.8ux %8.8ux %8.8ux\n", va, pa, x, tlbhi, tlbmi, tlblo);
205 if(stidget() != tp)
206 panic("mmuput stid:pid mismatch");
207 stidput(tp);
208 tlbwrx(x, tlbhi, tlbmi, tlblo);
209 putstlb(tp, TLBEPN(va), tlbmi, tlblo);
210
211 ctl = &page->cachectl[m->machno];
212 switch(*ctl){
213 case PG_TXTFLUSH:
214 dcflush(page->va, BY2PG);
215 icflush(page->va, BY2PG);
216 *ctl = PG_NOFLUSH;
217 break;
218 case PG_DATFLUSH:
219 dcflush(page->va, BY2PG);
220 *ctl = PG_NOFLUSH;
221 break;
222 case PG_NEWCOL:
223 // cleancache(); /* expensive, but fortunately not needed here */
224 *ctl = PG_NOFLUSH;
225 break;
226 }
227 splx(s);
228 }
229
230 /*
231 * Process must be splhi
232 */
233 static int
newtlbpid(Proc * p)234 newtlbpid(Proc *p)
235 {
236 int i, s;
237 Proc **h;
238
239 i = m->lastpid;
240 h = m->pidproc;
241 for(s = 0; s < NTLBPID; s++) {
242 i++;
243 if(i >= NTLBPID)
244 i = 1;
245 if(h[i] == nil)
246 break;
247 }
248
249 if(h[i] != nil){
250 purgetlb(i);
251 if(h[i] != nil)
252 panic("newtlb");
253 }
254
255 m->pidproc[i] = p;
256 p->mmupid = i;
257 m->lastpid = i;
258
259 return i;
260 }
261
262 static void
purgetlb(int pid)263 purgetlb(int pid)
264 {
265 int i;
266 Proc *sp, **pidproc;
267 Softtlb *entry, *etab;
268 u32int hi;
269
270 m->tlbpurge++;
271
272 /*
273 * find all pid entries that are no longer used by processes
274 */
275 pidproc = m->pidproc;
276 for(i=1; i<NTLBPID; i++) {
277 sp = pidproc[i];
278 if(sp != nil && sp->mmupid != i)
279 pidproc[i] = nil;
280 }
281
282 /*
283 * shoot down the one we want
284 */
285 sp = pidproc[pid];
286 if(sp != nil)
287 sp->mmupid = 0;
288 pidproc[pid] = nil;
289
290 /*
291 * clean out all dead pids from the stlb;
292 */
293 entry = m->stlb;
294 for(etab = &entry[STLBSIZE]; entry < etab; entry++)
295 if(pidproc[(entry->hi>>2)&0xFF] == nil){
296 entry->hi = 0;
297 entry->mid = 0;
298 entry->lo = 0;
299 }
300
301 /*
302 * clean up the hardware
303 */
304 for(i = 0; i <= m->utlbhi; i++){
305 hi = tlbrehi(i);
306 if((hi & TLBVALID) && pidproc[stidget()] == nil)
307 tlbwrx(i, 0, 0, 0);
308 }
309 stidput(pidget());
310 mbar();
311 barriers();
312 iccci();
313 }
314
315 /*
316 * return required size and alignment to map n bytes in a tlb entry
317 */
318 ulong
mmumapsize(ulong n)319 mmumapsize(ulong n)
320 {
321 ulong size;
322 int i;
323
324 size = 1024;
325 for(i = 0; i < 10 && size < n; i++)
326 size <<= 2;
327 return size;
328 }
329
330 /*
331 * map a physical addresses at pa to va, with the given attributes.
332 * the virtual address must not be mapped already.
333 * if va is nil, map it at pa in virtual space.
334 */
335 void*
kmapphys(uintptr va,uintptr pa,ulong nb,ulong attr,ulong le)336 kmapphys(uintptr va, uintptr pa, ulong nb, ulong attr, ulong le)
337 {
338 Mreg s;
339 int i;
340 ulong size;
341
342 if(va == 0)
343 va = pa; /* simplest is to use a 1-1 map */
344 size = 1024;
345 for(i = 0; i < 10 && size < nb; i++)
346 size <<= 2;
347 if(i >= 10)
348 return 0;
349 if(i == 6 || i == 8)
350 i++; /* skip sizes not implemented by hardware */
351 if(m->utlbhi <= Minutlb)
352 panic("kmapphys");
353
354 s = splhi();
355 stidput(0);
356 tlbwrx(m->utlbhi, va | (i<<4) | TLBVALID, pa, attr | le);
357 m->utlbhi--;
358 stidput(pidget());
359 splx(s);
360
361 if(DEBUG)
362 tlbdump("kmapphys");
363
364 return UINT2PTR(va);
365 }
366
367 /*
368 * return an uncached alias for the memory at a
369 */
370 void*
mmucacheinhib(void * a,ulong nb)371 mmucacheinhib(void* a, ulong nb)
372 {
373 uintptr pa;
374
375 if(a == nil)
376 return nil;
377 dcflush(PTR2UINT(a), nb);
378 pa = PADDR(a);
379 return kmapphys(KSEG1|pa, pa, nb, TLBWR | TLBI | TLBG, 0);
380 }
381
382 static void
putstlb(int pid,u32int va,u32int tlbmid,u32int tlblo)383 putstlb(int pid, u32int va, u32int tlbmid, u32int tlblo)
384 {
385 Softtlb *entry;
386
387 pid <<= 2;
388 entry = &m->stlb[((va>>12)^pid)&(STLBSIZE-1)];
389 entry->hi = va | pid;
390 entry->mid = tlbmid;
391 entry->lo = tlblo;
392 }
393
394 /*
395 * Return the number of bytes that can be accessed via KADDR(pa).
396 * If pa is not a valid argument to KADDR, return 0.
397 */
398 uintptr
cankaddr(uintptr pa)399 cankaddr(uintptr pa)
400 {
401 if( /* pa >= PHYSDRAM && */ pa < PHYSDRAM + 512*MiB)
402 return PHYSDRAM + 512*MiB - pa;
403 return 0;
404 }
405