1 #include "u.h"
2 #include "../port/lib.h"
3 #include "mem.h"
4 #include "dat.h"
5 #include "fns.h"
6 #include "io.h"
7
8 #include "arm.h"
9
10 #define L1X(va) FEXT((va), 20, 12)
11 #define L2X(va) FEXT((va), 12, 8)
12
13 enum {
14 L1lo = UZERO/MiB, /* L1X(UZERO)? */
15 L1hi = (USTKTOP+MiB-1)/MiB, /* L1X(USTKTOP+MiB-1)? */
16 };
17
18 #define ISHOLE(pte) ((pte) == 0)
19
20 /* dump level 1 page table at virtual addr l1 */
21 void
mmudump(PTE * l1)22 mmudump(PTE *l1)
23 {
24 int i, type, rngtype;
25 uintptr pa, startva, startpa;
26 uvlong va, endva;
27 PTE pte;
28
29 iprint("\n");
30 endva = startva = startpa = 0;
31 rngtype = 0;
32 /* dump first level of ptes */
33 for (va = i = 0; i < 4096; i++) {
34 pte = l1[i];
35 pa = pte & ~(MB - 1);
36 type = pte & (Fine|Section|Coarse);
37 if (ISHOLE(pte)) {
38 if (endva != 0) { /* open range? close it */
39 iprint("l1 maps va (%#lux-%#llux) -> pa %#lux type %#ux\n",
40 startva, endva-1, startpa, rngtype);
41 endva = 0;
42 }
43 } else {
44 if (endva == 0) { /* no open range? start one */
45 startva = va;
46 startpa = pa;
47 rngtype = type;
48 }
49 endva = va + MB; /* continue the open range */
50 }
51 va += MB;
52 }
53 if (endva != 0) /* close an open range */
54 iprint("l1 maps va (%#lux-%#llux) -> pa %#lux type %#ux\n",
55 startva, endva-1, startpa, rngtype);
56 }
57
58 #ifdef CRYPTOSANDBOX
59 extern uchar sandbox[64*1024+BY2PG];
60 #endif
61
62 /* identity map `mbs' megabytes from phys */
63 void
mmuidmap(uintptr phys,int mbs)64 mmuidmap(uintptr phys, int mbs)
65 {
66 PTE *l1;
67 uintptr pa, fpa;
68
69 pa = ttbget();
70 l1 = KADDR(pa);
71
72 for (fpa = phys; mbs-- > 0; fpa += MiB)
73 l1[L1X(fpa)] = fpa|Dom0|L1AP(Krw)|Section;
74 coherence();
75
76 mmuinvalidate();
77 cacheuwbinv();
78 l2cacheuwbinv();
79 }
80
81 void
mmuinit(void)82 mmuinit(void)
83 {
84 PTE *l1, *l2;
85 uintptr pa, i;
86
87 pa = ttbget();
88 l1 = KADDR(pa);
89
90 /*
91 * map high vectors to start of dram, but only 4K, not 1MB.
92 */
93 pa -= MACHSIZE+2*1024;
94 l2 = KADDR(pa);
95 memset(l2, 0, 1024);
96 /* vectors step on u-boot, but so do page tables */
97 l2[L2X(HVECTORS)] = PHYSDRAM|L2AP(Krw)|Small;
98 l1[L1X(HVECTORS)] = pa|Dom0|Coarse; /* vectors -> ttb-machsize-2k */
99
100 /* double map vectors at virtual 0 so reset will see them */
101 pa -= 1024;
102 l2 = KADDR(pa);
103 memset(l2, 0, 1024);
104 l2[L2X(0)] = PHYSDRAM|L2AP(Krw)|Small;
105 l1[L1X(0)] = pa|Dom0|Coarse;
106
107 /*
108 * set up L2 ptes for PHYSIO (i/o registers), with smaller pages to
109 * enable user-mode access to a few devices.
110 */
111 pa -= 1024;
112 l2 = KADDR(pa);
113 /* identity map by default */
114 for (i = 0; i < 1024/4; i++)
115 l2[L2X(VIRTIO + i*BY2PG)] = (PHYSIO + i*BY2PG)|L2AP(Krw)|Small;
116
117 #ifdef CRYPTOSANDBOX
118 /*
119 * rest is to let rae experiment with the crypto hardware
120 */
121 /* access to cycle counter */
122 l2[L2X(soc.clock)] = soc.clock | L2AP(Urw)|Small;
123 /* cesa registers; also visible in user space */
124 for (i = 0; i < 16; i++)
125 l2[L2X(soc.cesa + i*BY2PG)] = (soc.cesa + i*BY2PG) |
126 L2AP(Urw)|Small;
127 /* crypto sram; remapped to unused space and visible in user space */
128 l2[L2X(PHYSIO + 0xa0000)] = PHYSCESASRAM | L2AP(Urw)|Small;
129 /* 64k of scratch dram */
130 for (i = 0; i < 16; i++)
131 l2[L2X(PHYSIO + 0xb0000 + i*BY2PG)] =
132 (PADDR((uintptr)sandbox & ~(BY2PG-1)) + i*BY2PG) |
133 L2AP(Urw) | Small;
134 #endif
135
136 l1[L1X(VIRTIO)] = pa|Dom0|Coarse;
137 coherence();
138
139 mmuinvalidate();
140 cacheuwbinv();
141 l2cacheuwbinv();
142
143 m->mmul1 = l1;
144 // mmudump(l1); /* DEBUG. too early to print */
145 }
146
147 static void
mmul2empty(Proc * proc,int clear)148 mmul2empty(Proc* proc, int clear)
149 {
150 PTE *l1;
151 Page **l2, *page;
152
153 l1 = m->mmul1;
154 l2 = &proc->mmul2;
155 for(page = *l2; page != nil; page = page->next){
156 if(clear)
157 memset(UINT2PTR(page->va), 0, BY2PG);
158 l1[page->daddr] = Fault;
159 l2 = &page->next;
160 }
161 *l2 = proc->mmul2cache;
162 proc->mmul2cache = proc->mmul2;
163 proc->mmul2 = nil;
164 }
165
166 static void
mmul1empty(void)167 mmul1empty(void)
168 {
169 #ifdef notdef /* there's a bug in here */
170 PTE *l1;
171
172 /* clean out any user mappings still in l1 */
173 if(m->mmul1lo > L1lo){
174 if(m->mmul1lo == 1)
175 m->mmul1[L1lo] = Fault;
176 else
177 memset(&m->mmul1[L1lo], 0, m->mmul1lo*sizeof(PTE));
178 m->mmul1lo = L1lo;
179 }
180 if(m->mmul1hi < L1hi){
181 l1 = &m->mmul1[m->mmul1hi];
182 if((L1hi - m->mmul1hi) == 1)
183 *l1 = Fault;
184 else
185 memset(l1, 0, (L1hi - m->mmul1hi)*sizeof(PTE));
186 m->mmul1hi = L1hi;
187 }
188 #else
189 memset(&m->mmul1[L1lo], 0, (L1hi - L1lo)*sizeof(PTE));
190 #endif /* notdef */
191 }
192
193 void
mmuswitch(Proc * proc)194 mmuswitch(Proc* proc)
195 {
196 int x;
197 PTE *l1;
198 Page *page;
199
200 /* do kprocs get here and if so, do they need to? */
201 if(m->mmupid == proc->pid && !proc->newtlb)
202 return;
203 m->mmupid = proc->pid;
204
205 /* write back dirty and invalidate l1 caches */
206 cacheuwbinv();
207
208 if(proc->newtlb){
209 mmul2empty(proc, 1);
210 proc->newtlb = 0;
211 }
212
213 mmul1empty();
214
215 /* move in new map */
216 l1 = m->mmul1;
217 for(page = proc->mmul2; page != nil; page = page->next){
218 x = page->daddr;
219 l1[x] = PPN(page->pa)|Dom0|Coarse;
220 /* know here that L1lo < x < L1hi */
221 if(x+1 - m->mmul1lo < m->mmul1hi - x)
222 m->mmul1lo = x+1;
223 else
224 m->mmul1hi = x;
225 }
226
227 /* make sure map is in memory */
228 /* could be smarter about how much? */
229 cachedwbse(&l1[L1X(UZERO)], (L1hi - L1lo)*sizeof(PTE));
230 l2cacheuwbse(&l1[L1X(UZERO)], (L1hi - L1lo)*sizeof(PTE));
231
232 /* lose any possible stale tlb entries */
233 mmuinvalidate();
234
235 // mmudump(l1);
236 //print("mmuswitch l1lo %d l1hi %d %d\n",
237 // m->mmul1lo, m->mmul1hi, proc->kp);
238 //print("\n");
239 }
240
241 void
flushmmu(void)242 flushmmu(void)
243 {
244 int s;
245
246 s = splhi();
247 up->newtlb = 1;
248 mmuswitch(up);
249 splx(s);
250 }
251
252 void
mmurelease(Proc * proc)253 mmurelease(Proc* proc)
254 {
255 Page *page, *next;
256
257 /* write back dirty and invalidate l1 caches */
258 cacheuwbinv();
259
260 mmul2empty(proc, 0);
261 for(page = proc->mmul2cache; page != nil; page = next){
262 next = page->next;
263 if(--page->ref)
264 panic("mmurelease: page->ref %d", page->ref);
265 pagechainhead(page);
266 }
267 if(proc->mmul2cache && palloc.r.p)
268 wakeup(&palloc.r);
269 proc->mmul2cache = nil;
270
271 mmul1empty();
272
273 /* make sure map is in memory */
274 /* could be smarter about how much? */
275 cachedwbse(&m->mmul1[L1X(UZERO)], (L1hi - L1lo)*sizeof(PTE));
276 l2cacheuwbse(&m->mmul1[L1X(UZERO)], (L1hi - L1lo)*sizeof(PTE));
277
278 /* lose any possible stale tlb entries */
279 mmuinvalidate();
280 }
281
282 void
putmmu(uintptr va,uintptr pa,Page * page)283 putmmu(uintptr va, uintptr pa, Page* page)
284 {
285 int x;
286 Page *pg;
287 PTE *l1, *pte;
288
289 x = L1X(va);
290 l1 = &m->mmul1[x];
291 //print("putmmu(%#p, %#p, %#p) ", va, pa, page->pa);
292 //print("mmul1 %#p l1 %#p *l1 %#ux x %d pid %d\n",
293 // m->mmul1, l1, *l1, x, up->pid);
294 if(*l1 == Fault){
295 /* wasteful - l2 pages only have 256 entries - fix */
296 if(up->mmul2cache == nil){
297 /* auxpg since we don't need much? memset if so */
298 pg = newpage(1, 0, 0);
299 pg->va = VA(kmap(pg));
300 }
301 else{
302 pg = up->mmul2cache;
303 up->mmul2cache = pg->next;
304 memset(UINT2PTR(pg->va), 0, BY2PG);
305 }
306 pg->daddr = x;
307 pg->next = up->mmul2;
308 up->mmul2 = pg;
309
310 /* force l2 page to memory */
311 cachedwbse((void *)pg->va, BY2PG);
312 l2cacheuwbse((void *)pg->va, BY2PG);
313
314 *l1 = PPN(pg->pa)|Dom0|Coarse;
315 cachedwbse(l1, sizeof *l1);
316 l2cacheuwbse(l1, sizeof *l1);
317 //print("l1 %#p *l1 %#ux x %d pid %d\n", l1, *l1, x, up->pid);
318
319 if(x >= m->mmul1lo && x < m->mmul1hi){
320 if(x+1 - m->mmul1lo < m->mmul1hi - x)
321 m->mmul1lo = x+1;
322 else
323 m->mmul1hi = x;
324 }
325 }
326 pte = UINT2PTR(KADDR(PPN(*l1)));
327 //print("pte %#p index %ld %#ux\n", pte, L2X(va), *(pte+L2X(va)));
328
329 /* protection bits are
330 * PTERONLY|PTEVALID;
331 * PTEWRITE|PTEVALID;
332 * PTEWRITE|PTEUNCACHED|PTEVALID;
333 */
334 x = Small;
335 if(!(pa & PTEUNCACHED))
336 x |= Cached|Buffered;
337 if(pa & PTEWRITE)
338 x |= L2AP(Urw);
339 else
340 x |= L2AP(Uro);
341 pte[L2X(va)] = PPN(pa)|x;
342 cachedwbse(&pte[L2X(va)], sizeof pte[0]);
343 l2cacheuwbse(&pte[L2X(va)], sizeof pte[0]);
344
345 /* clear out the current entry */
346 mmuinvalidateaddr(PPN(va));
347
348 /*
349 * write back dirty entries - we need this because pio() in
350 * fault.c is writing via a different virt addr and won't clean
351 * its changes out of the dcache. Page coloring doesn't work
352 * on this mmu because the l1 virtual cache is set associative
353 * rather than direct mapped.
354 */
355 cachedwbinv();
356 if(page->cachectl[0] == PG_TXTFLUSH){
357 /* pio() sets PG_TXTFLUSH whenever a text pg has been written */
358 cacheiinv();
359 page->cachectl[0] = PG_NOFLUSH;
360 }
361 //print("putmmu %#p %#p %#p\n", va, pa, PPN(pa)|x);
362 }
363
364 void*
mmuuncache(void * v,usize size)365 mmuuncache(void* v, usize size)
366 {
367 int x;
368 PTE *pte;
369 uintptr va;
370
371 /*
372 * Simple helper for ucalloc().
373 * Uncache a Section, must already be
374 * valid in the MMU.
375 */
376 va = PTR2UINT(v);
377 assert(!(va & (1*MiB-1)) && size == 1*MiB);
378
379 x = L1X(va);
380 pte = &m->mmul1[x];
381 if((*pte & (Fine|Section|Coarse)) != Section)
382 return nil;
383 *pte &= ~(Cached|Buffered);
384 mmuinvalidateaddr(va);
385 cachedwbse(pte, 4);
386 l2cacheuwbse(pte, 4);
387
388 return v;
389 }
390
391 uintptr
mmukmap(uintptr va,uintptr pa,usize size)392 mmukmap(uintptr va, uintptr pa, usize size)
393 {
394 int x;
395 PTE *pte;
396
397 /*
398 * Stub.
399 */
400 assert(!(va & (1*MiB-1)) && !(pa & (1*MiB-1)) && size == 1*MiB);
401
402 x = L1X(va);
403 pte = &m->mmul1[x];
404 if(*pte != Fault)
405 return 0;
406 *pte = pa|Dom0|L1AP(Krw)|Section;
407 mmuinvalidateaddr(va);
408 cachedwbse(pte, 4);
409 l2cacheuwbse(pte, 4);
410
411 return va;
412 }
413
414 uintptr
mmukunmap(uintptr va,uintptr pa,usize size)415 mmukunmap(uintptr va, uintptr pa, usize size)
416 {
417 int x;
418 PTE *pte;
419
420 /*
421 * Stub.
422 */
423 assert(!(va & (1*MiB-1)) && !(pa & (1*MiB-1)) && size == 1*MiB);
424
425 x = L1X(va);
426 pte = &m->mmul1[x];
427 if(*pte != (pa|Dom0|L1AP(Krw)|Section))
428 return 0;
429 *pte = Fault;
430 mmuinvalidateaddr(va);
431 cachedwbse(pte, 4);
432 l2cacheuwbse(pte, 4);
433
434 return va;
435 }
436
437 /*
438 * Return the number of bytes that can be accessed via KADDR(pa).
439 * If pa is not a valid argument to KADDR, return 0.
440 */
441 uintptr
cankaddr(uintptr pa)442 cankaddr(uintptr pa)
443 {
444 if(pa < PHYSDRAM + memsize) /* assumes PHYSDRAM is 0 */
445 return PHYSDRAM + memsize - pa;
446 return 0;
447 }
448
449 /* from 386 */
450 void*
vmap(uintptr pa,usize size)451 vmap(uintptr pa, usize size)
452 {
453 uintptr pae, va;
454 usize o, osize;
455
456 /*
457 * XXX - replace with new vm stuff.
458 * Crock after crock - the first 4MB is mapped with 2MB pages
459 * so catch that and return good values because the current mmukmap
460 * will fail.
461 */
462 if(pa+size < 4*MiB)
463 return UINT2PTR(kseg0|pa);
464
465 osize = size;
466 o = pa & (BY2PG-1);
467 pa -= o;
468 size += o;
469 size = ROUNDUP(size, BY2PG);
470
471 va = kseg0|pa;
472 pae = mmukmap(va, pa, size);
473 if(pae == 0 || pae-size != pa)
474 panic("vmap(%#p, %ld) called from %#p: mmukmap fails %#p",
475 pa+o, osize, getcallerpc(&pa), pae);
476
477 return UINT2PTR(va+o);
478 }
479
480 /* from 386 */
481 void
vunmap(void * v,usize size)482 vunmap(void* v, usize size)
483 {
484 /*
485 * XXX - replace with new vm stuff.
486 * Can't do this until do real vmap for all space that
487 * might be used, e.g. stuff below 1MB which is currently
488 * mapped automagically at boot but that isn't used (or
489 * at least shouldn't be used) by the kernel.
490 upafree(PADDR(v), size);
491 */
492 USED(v, size);
493 }
494
495 /*
496 * Notes.
497 * Everything is in domain 0;
498 * domain 0 access bits in the DAC register are set
499 * to Client, which means access is controlled by the
500 * permission values set in the PTE.
501 *
502 * L1 access control for the kernel is set to 1 (RW,
503 * no user mode access);
504 * L2 access control for the kernel is set to 1 (ditto)
505 * for all 4 AP sets;
506 * L1 user mode access is never set;
507 * L2 access control for user mode is set to either
508 * 2 (RO) or 3 (RW) depending on whether text or data,
509 * for all 4 AP sets.
510 * (To get kernel RO set AP to 0 and S bit in control
511 * register c1).
512 * Coarse L1 page-tables are used. They have 256 entries
513 * and so consume 1024 bytes per table.
514 * Small L2 page-tables are used. They have 1024 entries
515 * and so consume 4096 bytes per table.
516 *
517 * 4KiB. That's the size of 1) a page, 2) the
518 * size allocated for an L2 page-table page (note only 1KiB
519 * is needed per L2 page - to be dealt with later) and
520 * 3) the size of the area in L1 needed to hold the PTEs
521 * to map 1GiB of user space (0 -> 0x3fffffff, 1024 entries).
522 */
523