1 /* $NetBSD: pmap_machdep.c,v 1.20 2024/01/01 17:18:02 skrll Exp $ */
2
3 /*
4 * Copyright (c) 2014, 2019, 2021 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Matt Thomas (of 3am Software Foundry), Maxime Villard, and
9 * Nick Hudson.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 #include "opt_riscv_debug.h"
34 #include "opt_multiprocessor.h"
35
36 #define __PMAP_PRIVATE
37
38 #include <sys/cdefs.h>
39 __RCSID("$NetBSD: pmap_machdep.c,v 1.20 2024/01/01 17:18:02 skrll Exp $");
40
41 #include <sys/param.h>
42 #include <sys/buf.h>
43 #include <sys/cpu.h>
44
45 #include <uvm/uvm.h>
46
47 #include <riscv/machdep.h>
48 #include <riscv/sysreg.h>
49
50 #ifdef VERBOSE_INIT_RISCV
51 #define VPRINTF(...) printf(__VA_ARGS__)
52 #else
53 #define VPRINTF(...) __nothing
54 #endif
55
56 vaddr_t pmap_direct_base __read_mostly;
57 vaddr_t pmap_direct_end __read_mostly;
58
59 void
pmap_zero_page(paddr_t pa)60 pmap_zero_page(paddr_t pa)
61 {
62 #ifdef _LP64
63 #ifdef PMAP_DIRECT_MAP
64 memset((void *)PMAP_DIRECT_MAP(pa), 0, PAGE_SIZE);
65 #else
66 #error "no direct map"
67 #endif
68 #else
69 KASSERT(false);
70 #endif
71 }
72
73 void
pmap_copy_page(paddr_t src,paddr_t dst)74 pmap_copy_page(paddr_t src, paddr_t dst)
75 {
76 #ifdef _LP64
77 #ifdef PMAP_DIRECT_MAP
78 memcpy((void *)PMAP_DIRECT_MAP(dst), (const void *)PMAP_DIRECT_MAP(src),
79 PAGE_SIZE);
80 #else
81 #error "no direct map"
82 #endif
83 #else
84 KASSERT(false);
85 #endif
86 }
87
88 struct vm_page *
pmap_md_alloc_poolpage(int flags)89 pmap_md_alloc_poolpage(int flags)
90 {
91
92 return uvm_pagealloc(NULL, 0, NULL, flags);
93 }
94
95 vaddr_t
pmap_md_map_poolpage(paddr_t pa,vsize_t len)96 pmap_md_map_poolpage(paddr_t pa, vsize_t len)
97 {
98 #ifdef _LP64
99 return PMAP_DIRECT_MAP(pa);
100 #else
101 panic("not supported");
102 #endif
103 }
104
105 void
pmap_md_unmap_poolpage(vaddr_t pa,vsize_t len)106 pmap_md_unmap_poolpage(vaddr_t pa, vsize_t len)
107 {
108 /* nothing to do */
109 }
110
111
112 bool
pmap_md_direct_mapped_vaddr_p(vaddr_t va)113 pmap_md_direct_mapped_vaddr_p(vaddr_t va)
114 {
115 #ifdef _LP64
116 return RISCV_DIRECTMAP_P(va);
117 #else
118 return false;
119 #endif
120 }
121
122 bool
pmap_md_io_vaddr_p(vaddr_t va)123 pmap_md_io_vaddr_p(vaddr_t va)
124 {
125 return false;
126 }
127
128 paddr_t
pmap_md_direct_mapped_vaddr_to_paddr(vaddr_t va)129 pmap_md_direct_mapped_vaddr_to_paddr(vaddr_t va)
130 {
131 #ifdef _LP64
132 #ifdef PMAP_DIRECT_MAP
133 return PMAP_DIRECT_UNMAP(va);
134 #else
135 KASSERT(false);
136 return 0;
137 #endif
138 #else
139 KASSERT(false);
140 return 0;
141 #endif
142 }
143
144 vaddr_t
pmap_md_direct_map_paddr(paddr_t pa)145 pmap_md_direct_map_paddr(paddr_t pa)
146 {
147 #ifdef _LP64
148 return PMAP_DIRECT_MAP(pa);
149 #else
150 panic("not supported");
151 #endif
152 }
153
154 void
pmap_md_init(void)155 pmap_md_init(void)
156 {
157 pmap_tlb_info_evcnt_attach(&pmap_tlb0_info);
158 }
159
160 bool
pmap_md_ok_to_steal_p(const uvm_physseg_t bank,size_t npgs)161 pmap_md_ok_to_steal_p(const uvm_physseg_t bank, size_t npgs)
162 {
163 return true;
164 }
165
166 #ifdef MULTIPROCESSOR
167 void
pmap_md_tlb_info_attach(struct pmap_tlb_info * ti,struct cpu_info * ci)168 pmap_md_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci)
169 {
170 }
171 #endif
172
173
174 void
pmap_md_xtab_activate(struct pmap * pmap,struct lwp * l)175 pmap_md_xtab_activate(struct pmap *pmap, struct lwp *l)
176 {
177 // UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
178
179 // struct cpu_info * const ci = curcpu();
180 struct pmap_tlb_info * const ti = cpu_tlb_info(ci);
181 struct pmap_asid_info * const pai = PMAP_PAI(pmap, ti);
182
183 uint64_t satp =
184 #ifdef _LP64
185 __SHIFTIN(SATP_MODE_SV39, SATP_MODE) |
186 #else
187 __SHIFTIN(SATP_MODE_SV32, SATP_MODE) |
188 #endif
189 __SHIFTIN(pai->pai_asid, SATP_ASID) |
190 __SHIFTIN(pmap->pm_md.md_ppn, SATP_PPN);
191
192 csr_satp_write(satp);
193
194 if (l && !tlbinfo_asids_p(ti)) {
195 tlb_invalidate_all();
196 }
197 }
198
199 void
pmap_md_xtab_deactivate(struct pmap * pmap)200 pmap_md_xtab_deactivate(struct pmap *pmap)
201 {
202
203 /* switch to kernel pmap */
204 pmap_md_xtab_activate(pmap_kernel(), NULL);
205 }
206
207 void
pmap_md_pdetab_init(struct pmap * pmap)208 pmap_md_pdetab_init(struct pmap *pmap)
209 {
210 KASSERT(pmap != NULL);
211
212 const vaddr_t pdetabva = (vaddr_t)pmap->pm_pdetab;
213 const paddr_t pdetabpa = pmap_md_direct_mapped_vaddr_to_paddr(pdetabva);
214 pmap->pm_md.md_ppn = pdetabpa >> PAGE_SHIFT;
215
216 /* XXXSB can we "pre-optimise" this by keeping a list of pdes to copy? */
217 /* XXXSB for relatively normal size memory (8gb) we only need 10-20ish ptes? */
218 /* XXXSB most (all?) of these ptes are in two consecutive ranges. */
219 for (size_t i = NPDEPG / 2; i < NPDEPG; ++i) {
220 /*
221 * XXXSB where/when do new entries in pmap_kernel()->pm_pdetab
222 * XXXSB get added to existing pmaps?
223 *
224 * pmap_growkernal doesn't have support for fixing up exiting
225 * pmaps. (yet)
226 *
227 * Various options:
228 *
229 * - do the x86 thing. maintain a list of pmaps and update them
230 * all in pmap_growkernel.
231 * - make sure the top level entries are populated and them simply
232 * copy "them all" here. If pmap_growkernel runs the new entries
233 * will become visible to all pmaps.
234 * - ...
235 */
236
237 /* XXXSB is this any faster than blindly copying all "high" entries? */
238 pd_entry_t pde = pmap_kernel()->pm_pdetab->pde_pde[i];
239
240 /* we might have leaf entries (direct map) as well as non-leaf */
241 if (pde) {
242 pmap->pm_pdetab->pde_pde[i] = pde;
243 }
244 }
245 }
246
247 void
pmap_md_pdetab_fini(struct pmap * pmap)248 pmap_md_pdetab_fini(struct pmap *pmap)
249 {
250
251 if (pmap == pmap_kernel())
252 return;
253 for (size_t i = NPDEPG / 2; i < NPDEPG; ++i) {
254 KASSERT(pte_invalid_pde() == 0);
255 pmap->pm_pdetab->pde_pde[i] = 0;
256 }
257 }
258
259 static void
pmap_md_grow(pmap_pdetab_t * ptb,vaddr_t va,vsize_t vshift,vsize_t * remaining)260 pmap_md_grow(pmap_pdetab_t *ptb, vaddr_t va, vsize_t vshift,
261 vsize_t *remaining)
262 {
263 KASSERT((va & (NBSEG - 1)) == 0);
264 #ifdef _LP64
265 const vaddr_t pdetab_mask = PMAP_PDETABSIZE - 1;
266 const vsize_t vinc = 1UL << vshift;
267
268 for (size_t i = (va >> vshift) & pdetab_mask;
269 i < PMAP_PDETABSIZE; i++, va += vinc) {
270 pd_entry_t * const pde_p =
271 &ptb->pde_pde[(va >> vshift) & pdetab_mask];
272
273 vaddr_t pdeva;
274 if (pte_pde_valid_p(*pde_p)) {
275 const paddr_t pa = pte_pde_to_paddr(*pde_p);
276 pdeva = pmap_md_direct_map_paddr(pa);
277 } else {
278 /*
279 * uvm_pageboot_alloc() returns a direct mapped address
280 */
281 pdeva = uvm_pageboot_alloc(PAGE_SIZE);
282 paddr_t pdepa = RISCV_KVA_TO_PA(pdeva);
283 *pde_p = pte_pde_pdetab(pdepa, true);
284 memset((void *)pdeva, 0, PAGE_SIZE);
285 }
286
287 if (vshift > SEGSHIFT) {
288 pmap_md_grow((pmap_pdetab_t *)pdeva, va,
289 vshift - SEGLENGTH, remaining);
290 } else {
291 if (*remaining > vinc)
292 *remaining -= vinc;
293 else
294 *remaining = 0;
295 }
296 if (*remaining == 0)
297 return;
298 }
299 #endif
300 }
301
302
303 void
pmap_bootstrap(vaddr_t vstart,vaddr_t vend)304 pmap_bootstrap(vaddr_t vstart, vaddr_t vend)
305 {
306 extern pmap_pdetab_t bootstrap_pde[PAGE_SIZE / sizeof(pd_entry_t)];
307
308 // pmap_pdetab_t * const kptb = &pmap_kern_pdetab;
309 pmap_t pm = pmap_kernel();
310
311 VPRINTF("common ");
312 pmap_bootstrap_common();
313
314 #ifdef MULTIPROCESSOR
315 VPRINTF("cpusets ");
316 struct cpu_info * const ci = curcpu();
317 kcpuset_create(&ci->ci_shootdowncpus, true);
318 #endif
319
320 VPRINTF("bs_pde %p ", bootstrap_pde);
321
322 // kend = (kend + 0x200000 - 1) & -0x200000;
323
324 /* Use the tables we already built in init_riscv() */
325 pm->pm_pdetab = bootstrap_pde;
326
327 /* Get the PPN for our page table root */
328 pm->pm_md.md_ppn = atop(KERN_VTOPHYS((vaddr_t)bootstrap_pde));
329
330 /* Setup basic info like pagesize=PAGE_SIZE */
331 // uvm_md_init();
332
333 /* init the lock */
334 // XXXNH per cpu?
335 pmap_tlb_info_init(&pmap_tlb0_info);
336
337 VPRINTF("ASID max %x ", pmap_tlb0_info.ti_asid_max);
338
339 #ifdef MULTIPROCESSOR
340 VPRINTF("kcpusets ");
341
342 kcpuset_create(&pm->pm_onproc, true);
343 kcpuset_create(&pm->pm_active, true);
344 KASSERT(pm->pm_onproc != NULL);
345 KASSERT(pm->pm_active != NULL);
346
347 kcpuset_set(pm->pm_onproc, cpu_index(ci));
348 kcpuset_set(pm->pm_active, cpu_index(ci));
349 #endif
350
351 VPRINTF("nkmempages ");
352 /*
353 * Compute the number of pages kmem_arena will have. This will also
354 * be called by uvm_km_bootstrap later, but that doesn't matter
355 */
356 kmeminit_nkmempages();
357
358 /* Get size of buffer cache and set an upper limit */
359 buf_setvalimit((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / 8);
360 vsize_t bufsz = buf_memcalc();
361 buf_setvalimit(bufsz);
362
363 vsize_t kvmsize = (VM_PHYS_SIZE + (ubc_nwins << ubc_winshift) +
364 bufsz + 16 * NCARGS + pager_map_size) +
365 /*(maxproc * UPAGES) + */nkmempages * NBPG;
366
367 #ifdef SYSVSHM
368 kvmsize += shminfo.shmall;
369 #endif
370
371 /* Calculate VA address space and roundup to NBSEG tables */
372 kvmsize = roundup(kvmsize, NBSEG);
373
374
375 /*
376 * Initialize `FYI' variables. Note we're relying on
377 * the fact that BSEARCH sorts the vm_physmem[] array
378 * for us. Must do this before uvm_pageboot_alloc()
379 * can be called.
380 */
381 pmap_limits.avail_start = ptoa(uvm_physseg_get_start(uvm_physseg_get_first()));
382 pmap_limits.avail_end = ptoa(uvm_physseg_get_end(uvm_physseg_get_last()));
383
384 /*
385 * Update the naive settings in pmap_limits to the actual KVA range.
386 */
387 pmap_limits.virtual_start = vstart;
388 pmap_limits.virtual_end = vend;
389
390 VPRINTF("limits: %" PRIxVADDR " - %" PRIxVADDR "\n", vstart, vend);
391
392 const vaddr_t kvmstart = vstart;
393 pmap_curmaxkvaddr = vstart + kvmsize;
394
395 VPRINTF("kva : %" PRIxVADDR " - %" PRIxVADDR "\n", kvmstart,
396 pmap_curmaxkvaddr);
397
398 pmap_md_grow(pmap_kernel()->pm_pdetab, kvmstart, XSEGSHIFT, &kvmsize);
399
400 /*
401 * Initialize the pools.
402 */
403
404 pool_init(&pmap_pmap_pool, PMAP_SIZE, 0, 0, 0, "pmappl",
405 &pool_allocator_nointr, IPL_NONE);
406
407 pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvpl",
408 #ifdef KASAN
409 NULL,
410 #else
411 &pmap_pv_page_allocator,
412 #endif
413 IPL_NONE);
414
415 // riscv_dcache_align
416 pmap_pvlist_lock_init(CACHE_LINE_SIZE);
417 }
418
419
420 vsize_t
pmap_kenter_range(vaddr_t va,paddr_t pa,vsize_t size,vm_prot_t prot,u_int flags)421 pmap_kenter_range(vaddr_t va, paddr_t pa, vsize_t size,
422 vm_prot_t prot, u_int flags)
423 {
424 extern pd_entry_t l1_pte[PAGE_SIZE / sizeof(pd_entry_t)];
425
426 vaddr_t sva = MEGAPAGE_TRUNC(va);
427 paddr_t spa = MEGAPAGE_TRUNC(pa);
428 const vaddr_t eva = MEGAPAGE_ROUND(va + size);
429 const vaddr_t pdetab_mask = PMAP_PDETABSIZE - 1;
430 const vsize_t vshift = SEGSHIFT;
431
432 while (sva < eva) {
433 const size_t sidx = (sva >> vshift) & pdetab_mask;
434
435 l1_pte[sidx] = PA_TO_PTE(spa) | PTE_KERN | PTE_HARDWIRED | PTE_RW;
436 spa += NBSEG;
437 sva += NBSEG;
438 }
439
440 return 0;
441 }
442