xref: /netbsd-src/sys/arch/riscv/riscv/pmap_machdep.c (revision 686b1ff1b40f2b7dd3dbdefeb64c511422977e43)
1 /* $NetBSD: pmap_machdep.c,v 1.20 2024/01/01 17:18:02 skrll Exp $ */
2 
3 /*
4  * Copyright (c) 2014, 2019, 2021 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Matt Thomas (of 3am Software Foundry), Maxime Villard, and
9  * Nick Hudson.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30  * POSSIBILITY OF SUCH DAMAGE.
31  */
32 
33 #include "opt_riscv_debug.h"
34 #include "opt_multiprocessor.h"
35 
36 #define	__PMAP_PRIVATE
37 
38 #include <sys/cdefs.h>
39 __RCSID("$NetBSD: pmap_machdep.c,v 1.20 2024/01/01 17:18:02 skrll Exp $");
40 
41 #include <sys/param.h>
42 #include <sys/buf.h>
43 #include <sys/cpu.h>
44 
45 #include <uvm/uvm.h>
46 
47 #include <riscv/machdep.h>
48 #include <riscv/sysreg.h>
49 
50 #ifdef VERBOSE_INIT_RISCV
51 #define	VPRINTF(...)	printf(__VA_ARGS__)
52 #else
53 #define	VPRINTF(...)	__nothing
54 #endif
55 
56 vaddr_t pmap_direct_base __read_mostly;
57 vaddr_t pmap_direct_end __read_mostly;
58 
59 void
pmap_zero_page(paddr_t pa)60 pmap_zero_page(paddr_t pa)
61 {
62 #ifdef _LP64
63 #ifdef PMAP_DIRECT_MAP
64 	memset((void *)PMAP_DIRECT_MAP(pa), 0, PAGE_SIZE);
65 #else
66 #error "no direct map"
67 #endif
68 #else
69 	KASSERT(false);
70 #endif
71 }
72 
73 void
pmap_copy_page(paddr_t src,paddr_t dst)74 pmap_copy_page(paddr_t src, paddr_t dst)
75 {
76 #ifdef _LP64
77 #ifdef PMAP_DIRECT_MAP
78 	memcpy((void *)PMAP_DIRECT_MAP(dst), (const void *)PMAP_DIRECT_MAP(src),
79 	    PAGE_SIZE);
80 #else
81 #error "no direct map"
82 #endif
83 #else
84 	KASSERT(false);
85 #endif
86 }
87 
88 struct vm_page *
pmap_md_alloc_poolpage(int flags)89 pmap_md_alloc_poolpage(int flags)
90 {
91 
92 	return uvm_pagealloc(NULL, 0, NULL, flags);
93 }
94 
95 vaddr_t
pmap_md_map_poolpage(paddr_t pa,vsize_t len)96 pmap_md_map_poolpage(paddr_t pa, vsize_t len)
97 {
98 #ifdef _LP64
99 	return PMAP_DIRECT_MAP(pa);
100 #else
101 	panic("not supported");
102 #endif
103 }
104 
105 void
pmap_md_unmap_poolpage(vaddr_t pa,vsize_t len)106 pmap_md_unmap_poolpage(vaddr_t pa, vsize_t len)
107 {
108 	/* nothing to do */
109 }
110 
111 
112 bool
pmap_md_direct_mapped_vaddr_p(vaddr_t va)113 pmap_md_direct_mapped_vaddr_p(vaddr_t va)
114 {
115 #ifdef _LP64
116 	return RISCV_DIRECTMAP_P(va);
117 #else
118 	return false;
119 #endif
120 }
121 
122 bool
pmap_md_io_vaddr_p(vaddr_t va)123 pmap_md_io_vaddr_p(vaddr_t va)
124 {
125 	return false;
126 }
127 
128 paddr_t
pmap_md_direct_mapped_vaddr_to_paddr(vaddr_t va)129 pmap_md_direct_mapped_vaddr_to_paddr(vaddr_t va)
130 {
131 #ifdef _LP64
132 #ifdef PMAP_DIRECT_MAP
133 	return PMAP_DIRECT_UNMAP(va);
134 #else
135 	KASSERT(false);
136 	return 0;
137 #endif
138 #else
139 	KASSERT(false);
140 	return 0;
141 #endif
142 }
143 
144 vaddr_t
pmap_md_direct_map_paddr(paddr_t pa)145 pmap_md_direct_map_paddr(paddr_t pa)
146 {
147 #ifdef _LP64
148 	return PMAP_DIRECT_MAP(pa);
149 #else
150 	panic("not supported");
151 #endif
152 }
153 
154 void
pmap_md_init(void)155 pmap_md_init(void)
156 {
157 	pmap_tlb_info_evcnt_attach(&pmap_tlb0_info);
158 }
159 
160 bool
pmap_md_ok_to_steal_p(const uvm_physseg_t bank,size_t npgs)161 pmap_md_ok_to_steal_p(const uvm_physseg_t bank, size_t npgs)
162 {
163 	return true;
164 }
165 
166 #ifdef MULTIPROCESSOR
167 void
pmap_md_tlb_info_attach(struct pmap_tlb_info * ti,struct cpu_info * ci)168 pmap_md_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci)
169 {
170 }
171 #endif
172 
173 
174 void
pmap_md_xtab_activate(struct pmap * pmap,struct lwp * l)175 pmap_md_xtab_activate(struct pmap *pmap, struct lwp *l)
176 {
177 //	UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
178 
179 //	struct cpu_info * const ci = curcpu();
180 	struct pmap_tlb_info * const ti = cpu_tlb_info(ci);
181 	struct pmap_asid_info * const pai = PMAP_PAI(pmap, ti);
182 
183 	uint64_t satp =
184 #ifdef _LP64
185 	    __SHIFTIN(SATP_MODE_SV39, SATP_MODE) |
186 #else
187 	    __SHIFTIN(SATP_MODE_SV32, SATP_MODE) |
188 #endif
189 	    __SHIFTIN(pai->pai_asid, SATP_ASID) |
190 	    __SHIFTIN(pmap->pm_md.md_ppn, SATP_PPN);
191 
192 	csr_satp_write(satp);
193 
194 	if (l && !tlbinfo_asids_p(ti)) {
195 		tlb_invalidate_all();
196 	}
197 }
198 
199 void
pmap_md_xtab_deactivate(struct pmap * pmap)200 pmap_md_xtab_deactivate(struct pmap *pmap)
201 {
202 
203 	/* switch to kernel pmap */
204 	pmap_md_xtab_activate(pmap_kernel(), NULL);
205 }
206 
207 void
pmap_md_pdetab_init(struct pmap * pmap)208 pmap_md_pdetab_init(struct pmap *pmap)
209 {
210 	KASSERT(pmap != NULL);
211 
212 	const vaddr_t pdetabva = (vaddr_t)pmap->pm_pdetab;
213 	const paddr_t pdetabpa = pmap_md_direct_mapped_vaddr_to_paddr(pdetabva);
214 	pmap->pm_md.md_ppn = pdetabpa >> PAGE_SHIFT;
215 
216 	/* XXXSB can we "pre-optimise" this by keeping a list of pdes to copy? */
217 	/* XXXSB for relatively normal size memory (8gb) we only need 10-20ish ptes? */
218 	/* XXXSB most (all?) of these ptes are  in two consecutive ranges. */
219 	for (size_t i = NPDEPG / 2; i < NPDEPG; ++i) {
220 		/*
221 		 * XXXSB where/when do new entries in pmap_kernel()->pm_pdetab
222 		 * XXXSB get added to existing pmaps?
223 		 *
224 		 * pmap_growkernal doesn't have support for fixing up exiting
225 		 * pmaps. (yet)
226 		 *
227 		 * Various options:
228 		 *
229 		 * - do the x86 thing. maintain a list of pmaps and update them
230 		 *   all in pmap_growkernel.
231 		 * - make sure the top level entries are populated and them simply
232 		 *   copy "them all" here.  If pmap_growkernel runs the new entries
233 		 *   will become visible to all pmaps.
234 		 * - ...
235 		 */
236 
237 		/* XXXSB is this any faster than blindly copying all "high" entries? */
238 		pd_entry_t pde = pmap_kernel()->pm_pdetab->pde_pde[i];
239 
240 		/*  we might have leaf entries (direct map) as well as non-leaf */
241 		if (pde) {
242 			pmap->pm_pdetab->pde_pde[i] = pde;
243 		}
244 	}
245 }
246 
247 void
pmap_md_pdetab_fini(struct pmap * pmap)248 pmap_md_pdetab_fini(struct pmap *pmap)
249 {
250 
251 	if (pmap == pmap_kernel())
252 		return;
253 	for (size_t i = NPDEPG / 2; i < NPDEPG; ++i) {
254 		KASSERT(pte_invalid_pde() == 0);
255 		pmap->pm_pdetab->pde_pde[i] = 0;
256 	}
257 }
258 
259 static void
pmap_md_grow(pmap_pdetab_t * ptb,vaddr_t va,vsize_t vshift,vsize_t * remaining)260 pmap_md_grow(pmap_pdetab_t *ptb, vaddr_t va, vsize_t vshift,
261     vsize_t *remaining)
262 {
263 	KASSERT((va & (NBSEG - 1)) == 0);
264 #ifdef _LP64
265 	const vaddr_t pdetab_mask = PMAP_PDETABSIZE - 1;
266 	const vsize_t vinc = 1UL << vshift;
267 
268 	for (size_t i = (va >> vshift) & pdetab_mask;
269 	    i < PMAP_PDETABSIZE; i++, va += vinc) {
270 		pd_entry_t * const pde_p =
271 		    &ptb->pde_pde[(va >> vshift) & pdetab_mask];
272 
273 		vaddr_t pdeva;
274 		if (pte_pde_valid_p(*pde_p)) {
275 			const paddr_t pa = pte_pde_to_paddr(*pde_p);
276 			pdeva = pmap_md_direct_map_paddr(pa);
277 		} else {
278 			/*
279 			 * uvm_pageboot_alloc() returns a direct mapped address
280 			 */
281 			pdeva = uvm_pageboot_alloc(PAGE_SIZE);
282 			paddr_t pdepa = RISCV_KVA_TO_PA(pdeva);
283 			*pde_p = pte_pde_pdetab(pdepa, true);
284 			memset((void *)pdeva, 0, PAGE_SIZE);
285 		}
286 
287 		if (vshift > SEGSHIFT) {
288 			pmap_md_grow((pmap_pdetab_t *)pdeva, va,
289 			    vshift - SEGLENGTH, remaining);
290 		} else {
291 			if (*remaining > vinc)
292 				*remaining -= vinc;
293 			else
294 				*remaining = 0;
295 		}
296 		if (*remaining == 0)
297 			return;
298 	}
299     #endif
300 }
301 
302 
303 void
pmap_bootstrap(vaddr_t vstart,vaddr_t vend)304 pmap_bootstrap(vaddr_t vstart, vaddr_t vend)
305 {
306 	extern pmap_pdetab_t bootstrap_pde[PAGE_SIZE / sizeof(pd_entry_t)];
307 
308 //	pmap_pdetab_t * const kptb = &pmap_kern_pdetab;
309 	pmap_t pm = pmap_kernel();
310 
311 	VPRINTF("common ");
312 	pmap_bootstrap_common();
313 
314 #ifdef MULTIPROCESSOR
315 	VPRINTF("cpusets ");
316 	struct cpu_info * const ci = curcpu();
317 	kcpuset_create(&ci->ci_shootdowncpus, true);
318 #endif
319 
320 	VPRINTF("bs_pde %p ", bootstrap_pde);
321 
322 //	kend = (kend + 0x200000 - 1) & -0x200000;
323 
324 	/* Use the tables we already built in init_riscv() */
325 	pm->pm_pdetab = bootstrap_pde;
326 
327 	/* Get the PPN for our page table root */
328 	pm->pm_md.md_ppn = atop(KERN_VTOPHYS((vaddr_t)bootstrap_pde));
329 
330 	/* Setup basic info like pagesize=PAGE_SIZE */
331 //	uvm_md_init();
332 
333 	/* init the lock */
334 	// XXXNH per cpu?
335 	pmap_tlb_info_init(&pmap_tlb0_info);
336 
337 	VPRINTF("ASID max %x ", pmap_tlb0_info.ti_asid_max);
338 
339 #ifdef MULTIPROCESSOR
340 	VPRINTF("kcpusets ");
341 
342 	kcpuset_create(&pm->pm_onproc, true);
343 	kcpuset_create(&pm->pm_active, true);
344 	KASSERT(pm->pm_onproc != NULL);
345 	KASSERT(pm->pm_active != NULL);
346 
347 	kcpuset_set(pm->pm_onproc, cpu_index(ci));
348 	kcpuset_set(pm->pm_active, cpu_index(ci));
349 #endif
350 
351 	VPRINTF("nkmempages ");
352 	/*
353 	 * Compute the number of pages kmem_arena will have.  This will also
354 	 * be called by uvm_km_bootstrap later, but that doesn't matter
355 	 */
356 	kmeminit_nkmempages();
357 
358 	/* Get size of buffer cache and set an upper limit */
359 	buf_setvalimit((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / 8);
360 	vsize_t bufsz = buf_memcalc();
361 	buf_setvalimit(bufsz);
362 
363 	vsize_t kvmsize = (VM_PHYS_SIZE + (ubc_nwins << ubc_winshift) +
364 	    bufsz + 16 * NCARGS + pager_map_size) +
365 	    /*(maxproc * UPAGES) + */nkmempages * NBPG;
366 
367 #ifdef SYSVSHM
368 	kvmsize += shminfo.shmall;
369 #endif
370 
371 	/* Calculate VA address space and roundup to NBSEG tables */
372 	kvmsize = roundup(kvmsize, NBSEG);
373 
374 
375 	/*
376 	 * Initialize `FYI' variables.	Note we're relying on
377 	 * the fact that BSEARCH sorts the vm_physmem[] array
378 	 * for us.  Must do this before uvm_pageboot_alloc()
379 	 * can be called.
380 	 */
381 	pmap_limits.avail_start = ptoa(uvm_physseg_get_start(uvm_physseg_get_first()));
382 	pmap_limits.avail_end = ptoa(uvm_physseg_get_end(uvm_physseg_get_last()));
383 
384 	/*
385 	 * Update the naive settings in pmap_limits to the actual KVA range.
386 	 */
387 	pmap_limits.virtual_start = vstart;
388 	pmap_limits.virtual_end = vend;
389 
390 	VPRINTF("limits: %" PRIxVADDR " - %" PRIxVADDR "\n", vstart, vend);
391 
392 	const vaddr_t kvmstart = vstart;
393 	pmap_curmaxkvaddr = vstart + kvmsize;
394 
395 	VPRINTF("kva   : %" PRIxVADDR " - %" PRIxVADDR "\n", kvmstart,
396 	    pmap_curmaxkvaddr);
397 
398 	pmap_md_grow(pmap_kernel()->pm_pdetab, kvmstart, XSEGSHIFT, &kvmsize);
399 
400 	/*
401 	 * Initialize the pools.
402 	 */
403 
404 	pool_init(&pmap_pmap_pool, PMAP_SIZE, 0, 0, 0, "pmappl",
405 	    &pool_allocator_nointr, IPL_NONE);
406 
407 	pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvpl",
408 #ifdef KASAN
409 	    NULL,
410 #else
411 	    &pmap_pv_page_allocator,
412 #endif
413 	    IPL_NONE);
414 
415 	// riscv_dcache_align
416 	pmap_pvlist_lock_init(CACHE_LINE_SIZE);
417 }
418 
419 
420 vsize_t
pmap_kenter_range(vaddr_t va,paddr_t pa,vsize_t size,vm_prot_t prot,u_int flags)421 pmap_kenter_range(vaddr_t va, paddr_t pa, vsize_t size,
422     vm_prot_t prot, u_int flags)
423 {
424 	extern pd_entry_t l1_pte[PAGE_SIZE / sizeof(pd_entry_t)];
425 
426 	vaddr_t sva = MEGAPAGE_TRUNC(va);
427 	paddr_t spa = MEGAPAGE_TRUNC(pa);
428 	const vaddr_t eva = MEGAPAGE_ROUND(va + size);
429 	const vaddr_t pdetab_mask = PMAP_PDETABSIZE - 1;
430 	const vsize_t vshift = SEGSHIFT;
431 
432 	while (sva < eva) {
433 		const size_t sidx = (sva >> vshift) & pdetab_mask;
434 
435 		l1_pte[sidx] = PA_TO_PTE(spa) | PTE_KERN | PTE_HARDWIRED | PTE_RW;
436 		spa += NBSEG;
437 		sva += NBSEG;
438 	}
439 
440 	return 0;
441 }
442