xref: /netbsd-src/sys/uvm/pmap/pmap_segtab.c (revision 4c3eb207d36f67d31994830c0a694161fc1ca39b)
1 /*	$NetBSD: pmap_segtab.c,v 1.31 2022/12/21 11:39:46 skrll Exp $	*/
2 
3 /*-
4  * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9  * NASA Ames Research Center and by Chris G. Demetriou.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30  * POSSIBILITY OF SUCH DAMAGE.
31  */
32 
33 /*
34  * Copyright (c) 1992, 1993
35  *	The Regents of the University of California.  All rights reserved.
36  *
37  * This code is derived from software contributed to Berkeley by
38  * the Systems Programming Group of the University of Utah Computer
39  * Science Department and Ralph Campbell.
40  *
41  * Redistribution and use in source and binary forms, with or without
42  * modification, are permitted provided that the following conditions
43  * are met:
44  * 1. Redistributions of source code must retain the above copyright
45  *    notice, this list of conditions and the following disclaimer.
46  * 2. Redistributions in binary form must reproduce the above copyright
47  *    notice, this list of conditions and the following disclaimer in the
48  *    documentation and/or other materials provided with the distribution.
49  * 3. Neither the name of the University nor the names of its contributors
50  *    may be used to endorse or promote products derived from this software
51  *    without specific prior written permission.
52  *
53  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
54  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
55  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
56  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
57  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
58  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
59  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
60  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
61  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
62  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
63  * SUCH DAMAGE.
64  *
65  *	@(#)pmap.c	8.4 (Berkeley) 1/26/94
66  */
67 
68 #include <sys/cdefs.h>
69 
70 __KERNEL_RCSID(0, "$NetBSD: pmap_segtab.c,v 1.31 2022/12/21 11:39:46 skrll Exp $");
71 
72 /*
73  *	Manages physical address maps.
74  *
75  *	In addition to hardware address maps, this
76  *	module is called upon to provide software-use-only
77  *	maps which may or may not be stored in the same
78  *	form as hardware maps.  These pseudo-maps are
79  *	used to store intermediate results from copy
80  *	operations to and from address spaces.
81  *
82  *	Since the information managed by this module is
83  *	also stored by the logical address mapping module,
84  *	this module may throw away valid virtual-to-physical
85  *	mappings at almost any time.  However, invalidations
86  *	of virtual-to-physical mappings must be done as
87  *	requested.
88  *
89  *	In order to cope with hardware architectures which
90  *	make virtual-to-physical map invalidates expensive,
91  *	this module may delay invalidate or reduced protection
92  *	operations until such time as they are actually
93  *	necessary.  This module is given full information as
94  *	to which processors are currently using which maps,
95  *	and to when physical maps must be made correct.
96  */
97 
98 #define __PMAP_PRIVATE
99 
100 #include "opt_multiprocessor.h"
101 
102 #include <sys/param.h>
103 
104 #include <sys/atomic.h>
105 #include <sys/mutex.h>
106 #include <sys/proc.h>
107 #include <sys/systm.h>
108 
109 #include <uvm/uvm.h>
110 #include <uvm/pmap/pmap.h>
111 
112 #if defined(XSEGSHIFT) && XSEGSHIFT == SEGSHIFT
113 #undef XSEGSHIFT
114 #undef XSEGLENGTH
115 #undef NBXSEG
116 #undef NXSEGPG
117 #endif
118 
119 #define MULT_CTASSERT(a,b)	__CTASSERT((a) < (b) || ((a) % (b) == 0))
120 
121 __CTASSERT(sizeof(pmap_ptpage_t) == NBPG);
122 
123 #if defined(PMAP_HWPAGEWALKER)
124 #ifdef _LP64
125 MULT_CTASSERT(PMAP_PDETABSIZE, NPDEPG);
126 MULT_CTASSERT(NPDEPG, PMAP_PDETABSIZE);
127 #endif /* _LP64 */
128 MULT_CTASSERT(sizeof(pmap_pdetab_t *), sizeof(pd_entry_t));
129 MULT_CTASSERT(sizeof(pd_entry_t), sizeof(pmap_pdetab_t));
130 
131 #if 0
132 #ifdef _LP64
133 static const bool separate_pdetab_root_p = NPDEPG != PMAP_PDETABSIZE;
134 #else
135 static const bool separate_pdetab_root_p = true;
136 #endif /* _LP64 */
137 #endif
138 
139 typedef struct {
140 	pmap_pdetab_t *free_pdetab0;	/* free list kept locally */
141 	pmap_pdetab_t *free_pdetab;	/* free list kept locally */
142 #ifdef DEBUG
143 	uint32_t nget;
144 	uint32_t nput;
145 	uint32_t npage;
146 #define	PDETAB_ADD(n, v)	(pmap_segtab_info.pdealloc.n += (v))
147 #else
148 #define	PDETAB_ADD(n, v)	((void) 0)
149 #endif /* DEBUG */
150 } pmap_pdetab_alloc_t;
151 #endif /* PMAP_HWPAGEWALKER */
152 
153 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE)
154 #ifdef _LP64
155 __CTASSERT(NSEGPG >= PMAP_SEGTABSIZE);
156 __CTASSERT(NSEGPG % PMAP_SEGTABSIZE == 0);
157 #endif
158 __CTASSERT(NBPG >= sizeof(pmap_segtab_t));
159 
160 typedef struct {
161 	pmap_segtab_t *free_segtab0;	/* free list kept locally */
162 	pmap_segtab_t *free_segtab;	/* free list kept locally */
163 #ifdef DEBUG
164 	uint32_t nget;
165 	uint32_t nput;
166 	uint32_t npage;
167 #define	SEGTAB_ADD(n, v)	(pmap_segtab_info.segalloc.n += (v))
168 #else
169 #define	SEGTAB_ADD(n, v)	((void) 0)
170 #endif
171 } pmap_segtab_alloc_t;
172 #endif /* !PMAP_HWPAGEWALKER || !PMAP_MAP_PDETABPAGE */
173 
174 struct pmap_segtab_info {
175 #if defined(PMAP_HWPAGEWALKER)
176 	pmap_pdetab_alloc_t pdealloc;
177 #endif
178 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE)
179 	pmap_segtab_alloc_t segalloc;
180 #endif
181 #ifdef PMAP_PPG_CACHE
182 	struct pgflist ptp_pgflist;	/* Keep a list of idle page tables. */
183 #endif
184 } pmap_segtab_info = {
185 #ifdef PMAP_PPG_CACHE
186 	.ptp_pgflist = LIST_HEAD_INITIALIZER(pmap_segtab_info.ptp_pgflist),
187 #endif
188 };
189 
190 kmutex_t pmap_segtab_lock __cacheline_aligned;
191 
192 #ifndef PMAP_HWPAGEWALKER
193 /*
194  * Check that a seg_ppg[] array is empty.
195  *
196  * This is used when allocating or freeing a pmap_segtab_t.  The stb
197  * should be unused -- meaning, none of the seg_ppg[] pointers are
198  * not NULL, as it transitions from either freshly allocated segtab from
199  * pmap pool, an unused allocated page segtab alloc from the SMP case,
200  * where two CPUs attempt to allocate the same underlying segtab, the
201  * release of a segtab entry to the freelist, or for SMP, where reserve
202  * also frees a freshly allocated but unused entry.
203  */
204 static void
205 pmap_check_stb(pmap_segtab_t *stb, const char *caller, const char *why)
206 {
207 #ifdef DEBUG
208 	for (size_t i = 0; i < PMAP_SEGTABSIZE; i++) {
209 		if (stb->seg_ppg[i] != NULL) {
210 #define DEBUG_NOISY
211 #ifdef DEBUG_NOISY
212 			UVMHIST_FUNC(__func__);
213 			UVMHIST_CALLARGS(pmapxtabhist, "stb=%#jx",
214 			    (uintptr_t)stb, 0, 0, 0);
215 			for (size_t j = i; j < PMAP_SEGTABSIZE; j++)
216 				if (stb->seg_ppg[j] != NULL)
217 					printf("%s: stb->seg_ppg[%zu] = %p\n",
218 					    caller, j, stb->seg_ppg[j]);
219 #endif
220 			panic("%s: pm_segtab.seg_ppg[%zu] != 0 (%p): %s",
221 			    caller, i, stb->seg_ppg[i], why);
222 		}
223 	}
224 #endif
225 }
226 #endif /* PMAP_HWPAGEWALKER */
227 
228 static inline struct vm_page *
229 pmap_pte_pagealloc(void)
230 {
231 	struct vm_page *pg;
232 
233 	pg = pmap_md_alloc_poolpage(UVM_PGA_ZERO | UVM_PGA_USERESERVE);
234 	if (pg) {
235 #ifdef UVM_PAGE_TRKOWN
236 		pg->owner_tag = NULL;
237 #endif
238 		UVM_PAGE_OWN(pg, "pmap-ptp");
239 	}
240 
241 	return pg;
242 }
243 
244 #if defined(PMAP_HWPAGEWALKER) && defined(PMAP_MAP_PDETABPAGE)
245 static vaddr_t
246 pmap_pde_to_va(pd_entry_t pde)
247 {
248 	if (!pte_pde_valid_p(pde))
249 		return 0;
250 
251 	paddr_t pa = pte_pde_to_paddr(pde);
252 	return pmap_md_direct_map_paddr(pa);
253 }
254 
255 #ifdef _LP64
256 static pmap_pdetab_t *
257 pmap_pde_to_pdetab(pd_entry_t pde)
258 {
259 
260 	return (pmap_pdetab_t *)pmap_pde_to_va(pde);
261 }
262 #endif
263 
264 static pmap_ptpage_t *
265 pmap_pde_to_ptpage(pd_entry_t pde)
266 {
267 
268 	return (pmap_ptpage_t *)pmap_pde_to_va(pde);
269 }
270 #endif
271 
272 #ifdef _LP64
273 __CTASSERT((XSEGSHIFT - SEGSHIFT) % (PGSHIFT-3) == 0);
274 #endif
275 
276 static inline pmap_ptpage_t *
277 pmap_ptpage(struct pmap *pmap, vaddr_t va)
278 {
279 #if defined(PMAP_HWPAGEWALKER) && defined(PMAP_MAP_PDETABPAGE)
280 	vaddr_t pdetab_mask = PMAP_PDETABSIZE - 1;
281 	pmap_pdetab_t *ptb = pmap->pm_pdetab;
282 
283 //	UVMHIST_LOG(pmaphist, "pm_pdetab %#jx", ptb, 0, 0, 0);
284 
285 	KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va));
286 
287 #ifdef _LP64
288 	for (size_t segshift = XSEGSHIFT;
289 	    segshift > SEGSHIFT;
290 	    segshift -= PGSHIFT - 3, pdetab_mask = NSEGPG - 1) {
291 		ptb = pmap_pde_to_pdetab(ptb->pde_pde[(va >> segshift) & pdetab_mask]);
292 		if (ptb == NULL)
293 			return NULL;
294 	}
295 #endif
296 	return pmap_pde_to_ptpage(ptb->pde_pde[(va >> SEGSHIFT) & pdetab_mask]);
297 #else
298 	vaddr_t segtab_mask = PMAP_SEGTABSIZE - 1;
299 	pmap_segtab_t *stb = pmap->pm_segtab;
300 
301 	KASSERTMSG(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va),
302 	    "pmap %p va %#" PRIxVADDR, pmap, va);
303 #ifdef _LP64
304 	for (size_t segshift = XSEGSHIFT;
305 	    segshift > SEGSHIFT;
306 	    segshift -= PGSHIFT - 3, segtab_mask = NSEGPG - 1) {
307 		stb = stb->seg_seg[(va >> segshift) & segtab_mask];
308 		if (stb == NULL)
309 			return NULL;
310 	}
311 #endif
312 	return stb->seg_ppg[(va >> SEGSHIFT) & segtab_mask];
313 #endif
314 }
315 
316 #if defined(PMAP_HWPAGEWALKER)
317 bool
318 pmap_pdetab_fixup(struct pmap *pmap, vaddr_t va)
319 {
320 	struct pmap * const kpm = pmap_kernel();
321 	pmap_pdetab_t * const kptb = kpm->pm_pdetab;
322 	pmap_pdetab_t * const uptb = pmap->pm_pdetab;
323 	size_t idx = PMAP_PDETABSIZE - 1;
324 #if !defined(PMAP_MAP_PDETABPAGE)
325 	__CTASSERT(PMAP_PDETABSIZE == PMAP_SEGTABSIZE);
326 	pmap_segtab_t * const kstb = &pmap_kern_segtab;
327 	pmap_segtab_t * const ustb = pmap->pm_segtab;
328 #endif
329 
330 	// Regardless of how many levels deep this page table is, we only
331 	// need to verify the first level PDEs match up.
332 #ifdef XSEGSHIFT
333 	idx &= va >> XSEGSHIFT;
334 #else
335 	idx &= va >> SEGSHIFT;
336 #endif
337 	if (uptb->pde_pde[idx] != kptb->pde_pde[idx]) {
338 		pte_pde_set(&uptb->pde_pde[idx], kptb->pde_pde[idx]);
339 #if !defined(PMAP_MAP_PDETABPAGE)
340 		ustb->seg_seg[idx] = kstb->seg_seg[idx]; // copy KVA of PTP
341 #endif
342 		return true;
343 	}
344 	return false;
345 }
346 #endif /* PMAP_HWPAGEWALKER */
347 
348 
349 static void
350 pmap_page_attach(pmap_t pmap, vaddr_t kva, struct vm_page *pg,
351     struct pglist *pglist, voff_t off)
352 {
353 	UVMHIST_FUNC(__func__);
354 	UVMHIST_CALLARGS(pmapxtabhist, "pm %#jx kva %#jx pg %#jx list %#jx",
355 	    (uintptr_t)pmap, (uintptr_t)kva, (uintptr_t)pg, (uintptr_t)pglist);
356 
357 	struct uvm_object * const uobj = &pmap->pm_uobject;
358 	if (pg == NULL) {
359 		paddr_t pa;
360 
361 		bool ok __diagused = pmap_extract(pmap_kernel(), kva, &pa);
362 		KASSERT(ok);
363 
364 		pg = PHYS_TO_VM_PAGE(pa);
365 		KASSERT(pg != NULL);
366 	}
367 
368 	UVMHIST_LOG(pmapxtabhist, "kva %#jx uobj %#jx pg %#jx list %#jx",
369 	    (uintptr_t)kva, (uintptr_t)uobj, (uintptr_t)pg, (uintptr_t)pglist);
370 
371 	pmap_lock(pmap);
372 	TAILQ_INSERT_TAIL(pglist, pg, pageq.queue);
373 	uobj->uo_npages++;
374 	pmap_unlock(pmap);
375 
376 	/*
377 	 * Now set each vm_page that maps this page to point to the
378 	 * pmap and set the offset to what we want.
379 	 */
380 	KASSERTMSG(pg->uobject == NULL, "pg %p pg->uobject %p", pg, pg->uobject);
381 	pg->uobject = uobj;
382 	pg->offset = off;
383 }
384 
385 static struct vm_page *
386 pmap_page_detach(pmap_t pmap, struct pglist *list, vaddr_t va)
387 {
388 	UVMHIST_FUNC(__func__);
389 	UVMHIST_CALLARGS(pmapxtabhist, "pm %#jx kva %#jx list %#jx",
390 	    (uintptr_t)pmap, (uintptr_t)va, (uintptr_t)list, 0);
391 
392 	paddr_t pa;
393 	bool ok __diagused = pmap_extract(pmap_kernel(), va, &pa);
394 	KASSERT(ok);
395 
396 	struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
397 	struct uvm_object * const uobj = &pmap->pm_uobject;
398 
399 	UVMHIST_LOG(pmapxtabhist, "kva %#jx uobj %#jx pg %#jx list %#jx",
400 	    (uintptr_t)va, (uintptr_t)uobj, (uintptr_t)pg, (uintptr_t)list);
401 
402 	KASSERTMSG(pg->uobject == uobj, "pg->uobject %p vs uobj %p",
403 	    pg->uobject, uobj);
404 
405 	pmap_lock(pmap);
406 	TAILQ_REMOVE(list, pg, pageq.queue);
407 	uobj->uo_npages--;
408 	pmap_unlock(pmap);
409 
410 	pg->uobject = NULL;
411 	pg->offset = 0;
412 
413 	return pg;
414 }
415 
416 #ifndef PMAP_PPG_CACHE
417 static void
418 pmap_segtab_pagefree(pmap_t pmap, struct pglist *list, vaddr_t kva, size_t size)
419 {
420 #ifdef PMAP_MAP_PTEPAGE
421 	UVMHIST_FUNC(__func__);
422 	UVMHIST_CALLARGS(pmapxtabhist, "pm %#jx list %#jx kva %#jx size %#jx",
423 	    (uintptr_t)pmap, (uintptr_t)list, kva, size);
424 	KASSERT(size == PAGE_SIZE);
425 	if (size == PAGE_SIZE) {
426 		UVMHIST_LOG(pmapxtabhist, "about to detach (kva %#jx)",
427 		    kva, 0, 0, 0);
428 		uvm_pagefree(pmap_page_detach(pmap, list, kva));
429 		return;
430 	}
431 #endif
432 	for (size_t i = 0; i < size; i += PAGE_SIZE) {
433 		(void)pmap_page_detach(pmap, list, kva + i);
434 	}
435 
436 	uvm_km_free(kernel_map, kva, size, UVM_KMF_WIRED);
437 }
438 #endif
439 
440 pt_entry_t *
441 pmap_pte_lookup(pmap_t pmap, vaddr_t va)
442 {
443 	pmap_ptpage_t * const ppg = pmap_ptpage(pmap, va);
444 	if (ppg == NULL)
445 		return NULL;
446 
447 	const size_t pte_idx = (va >> PGSHIFT) & (NPTEPG - 1);
448 
449 	return ppg->ppg_ptes + pte_idx;
450 }
451 
452 
453 static pmap_ptpage_t *
454 pmap_ptpage_alloc(pmap_t pmap, int flags, paddr_t *pa_p)
455 {
456 	UVMHIST_FUNC(__func__);
457 	UVMHIST_CALLARGS(pmapxtabhist, "pm %#jx flags %#jx pa_p %#jx", (uintptr_t)pmap,
458 	    (uintptr_t)flags, (uintptr_t)pa_p, 0);
459 
460 	pmap_ptpage_t *ppg = NULL;
461 
462 #ifdef PMAP_MAP_PTEPAGE
463 	struct vm_page *pg = NULL;
464 	paddr_t pa;
465 #ifdef PMAP_PPG_CACHE
466 	ppg = pmap_pgcache_alloc(&pmap_segtab_info.ppg_flist);
467 #endif
468 	if (ppg == NULL) {
469 		pg = pmap_pte_pagealloc();
470 		if (pg == NULL) {
471 			if (flags & PMAP_CANFAIL)
472 				return NULL;
473 			panic("%s: cannot allocate page table page ",
474 			    __func__);
475 		}
476 		pa = VM_PAGE_TO_PHYS(pg);
477 		ppg = (pmap_ptpage_t *)PMAP_MAP_PTEPAGE(pa);
478 	} else {
479 		bool ok __diagused = pmap_extract(pmap_kernel(), (vaddr_t)ppg, &pa);
480 		KASSERT(ok);
481 	}
482 
483 	UVMHIST_LOG(pmapxtabhist, "about to attach",  0, 0, 0, 0);
484 	pmap_page_attach(pmap, (vaddr_t)ppg, pg, &pmap->pm_ppg_list, 0);
485 
486 	*pa_p = pa;
487 #else
488 	vaddr_t kva = uvm_km_alloc(kernel_map, PAGE_SIZE, PAGE_SIZE,
489 	    UVM_KMF_WIRED | UVM_KMF_WAITVA
490 	    | (flags & PMAP_CANFAIL ? UVM_KMF_CANFAIL : 0));
491 	if (kva == 0) {
492 		if (flags & PMAP_CANFAIL)
493 			return NULL;
494 		panic("%s: cannot allocate page table page", __func__);
495 	}
496 	UVMHIST_LOG(pmapxtabhist, "about to attach",  0, 0, 0, 0);
497 	pmap_page_attach(pmap, kva, NULL, &pmap->pm_ppg_list, 0);
498 	ppg = (pmap_ptpage_t *)kva;
499 #endif
500 
501 	UVMHIST_LOG(pmapxtabhist, "... ppg %#jx", (uintptr_t)ppg, 0, 0, 0);
502 
503 	return ppg;
504 }
505 
506 static void
507 pmap_ptpage_free(pmap_t pmap, pmap_ptpage_t *ppg, const char *caller)
508 {
509 	UVMHIST_FUNC(__func__);
510 	UVMHIST_CALLARGS(pmapxtabhist, "pm %#jx va %#jx", (uintptr_t)pmap,
511 	    (uintptr_t)ppg, 0, 0);
512 
513 	const vaddr_t kva = (vaddr_t)ppg;
514 	/*
515 	 * All pte arrays should be page aligned.
516 	 */
517 	if ((kva & PAGE_MASK) != 0) {
518 		panic("%s: pte entry at %p not page aligned", caller, ppg);
519 	}
520 
521 #ifdef DEBUG
522 	for (size_t j = 0; j < NPTEPG; j++) {
523 		if (ppg->ppg_ptes[j] != 0) {
524 			UVMHIST_LOG(pmapxtabhist,
525 			    "pte entry %#jx not 0 (%#jx)",
526 			    (uintptr_t)&ppg->ppg_ptes[j],
527 			    (uintptr_t)ppg->ppg_ptes[j], 0, 0);
528 			for (size_t i = j + 1; i < NPTEPG; i++)
529 				if (ppg->ppg_ptes[i] != 0)
530 					UVMHIST_LOG(pmapxtabhist,
531 					    "pte[%zu] = %#"PRIxPTE,
532 					    i, ppg->ppg_ptes[i], 0, 0);
533 
534 			panic("%s: pte entry at %p not 0 (%#" PRIxPTE ")",
535 			    __func__, &ppg->ppg_ptes[j],
536 			    ppg->ppg_ptes[j]);
537 		}
538 	}
539 #endif
540 	//pmap_md_vca_clean(pg, (vaddr_t)ppg, NBPG);
541 #ifdef PMAP_PPG_CACHE
542 	UVMHIST_LOG(pmapxtabhist, "about to detach",  0, 0, 0, 0);
543 	pmap_page_detach(pmap, &pmap->pm_ppg_list, kva);
544 	pmap_segtab_pagecache(&pmap_segtab_info.ppg_flist, ppg);
545 #else
546 	pmap_segtab_pagefree(pmap, &pmap->pm_ppg_list, kva, PAGE_SIZE);
547 #endif /* PMAP_PPG_CACHE */
548 }
549 
550 
551 #if defined(PMAP_HWPAGEWALKER) && defined(PMAP_MAP_PDETABPAGE)
552 
553 static pmap_pdetab_t *
554 pmap_pdetab_alloc(struct pmap *pmap)
555 {
556 	UVMHIST_FUNC(__func__);
557 	UVMHIST_CALLARGS(pmapxtabhist, "pm %#jx", (uintptr_t)pmap, 0, 0, 0);
558 
559 	pmap_pdetab_t *ptb;
560 #ifdef KERNHIST
561 	bool found_on_freelist = false;
562 #endif
563 
564  again:
565 	mutex_spin_enter(&pmap_segtab_lock);
566 	UVMHIST_LOG(pmapxtabhist, "free_pdetab %#jx",
567 	    (uintptr_t)pmap_segtab_info.pdealloc.free_pdetab, 0, 0, 0);
568 	if (__predict_true((ptb = pmap_segtab_info.pdealloc.free_pdetab) != NULL)) {
569 		pmap_segtab_info.pdealloc.free_pdetab = ptb->pde_next;
570 
571 		UVMHIST_LOG(pmapxtabhist, "freelist ptb=%#jx",
572 		    (uintptr_t)ptb, 0, 0, 0);
573 
574 		PDETAB_ADD(nget, 1);
575 		ptb->pde_next = NULL;
576 #ifdef KERNHIST
577 		found_on_freelist = true;
578 #endif
579 	}
580 	mutex_spin_exit(&pmap_segtab_lock);
581 
582 	struct vm_page *ptb_pg = NULL;
583 	if (__predict_false(ptb == NULL)) {
584 		ptb_pg = pmap_pte_pagealloc();
585 
586 		UVMHIST_LOG(pmapxtabhist, "ptb_pg=%#jx",
587 		    (uintptr_t)ptb_pg, 0, 0, 0);
588 		if (__predict_false(ptb_pg == NULL)) {
589 			/*
590 			 * XXX What else can we do?  Could we deadlock here?
591 			 */
592 			uvm_wait("pdetab");
593 			goto again;
594 		}
595 
596 		UVMHIST_LOG(pmapxtabhist, "ptb_pg=%#jx 2",
597 		    (uintptr_t)ptb_pg, 0, 0, 0);
598 		PDETAB_ADD(npage, 1);
599 		const paddr_t ptb_pa = VM_PAGE_TO_PHYS(ptb_pg);
600 		UVMHIST_LOG(pmapxtabhist, "ptb_pa=%#jx",  (uintptr_t)ptb_pa, 0, 0, 0);
601 		ptb = (pmap_pdetab_t *)PMAP_MAP_PDETABPAGE(ptb_pa);
602 		UVMHIST_LOG(pmapxtabhist, "new ptb=%#jx", (uintptr_t)ptb, 0,
603 		    0, 0);
604 
605 		if (pte_invalid_pde() != 0) {
606 			for (size_t i = 0; i < NPDEPG; i++) {
607 				ptb->pde_pde[i] = pte_invalid_pde();
608 			}
609 		}
610 	}
611 
612 	UVMHIST_LOG(pmapxtabhist, "about to attach",  0, 0, 0, 0);
613 	pmap_page_attach(pmap, (vaddr_t)ptb, ptb_pg, &pmap->pm_pdetab_list, 0);
614 
615 	UVMHIST_LOG(pmapxtabhist, "... ptb %#jx found on freelist %d",
616 	    (uintptr_t)ptb, found_on_freelist, 0, 0);
617 
618 	return ptb;
619 }
620 
621 
622 #else
623 /*
624  *	Create and return a physical map.
625  *
626  *	If the size specified for the map
627  *	is zero, the map is an actual physical
628  *	map, and may be referenced by the
629  *	hardware.
630  *
631  *	If the size specified is non-zero,
632  *	the map will be used in software only, and
633  *	is bounded by that size.
634  */
635 static pmap_segtab_t *
636 pmap_segtab_alloc(struct pmap *pmap)
637 {
638 	UVMHIST_FUNC(__func__);
639 	UVMHIST_CALLARGS(pmapxtabhist, "pm %#jx", (uintptr_t)pmap, 0, 0, 0);
640 
641 	pmap_segtab_t *stb;
642 	bool found_on_freelist = false;
643 
644  again:
645 	mutex_spin_enter(&pmap_segtab_lock);
646 	if (__predict_true((stb = pmap_segtab_info.segalloc.free_segtab) != NULL)) {
647 		pmap_segtab_info.segalloc.free_segtab = stb->seg_next;
648 		SEGTAB_ADD(nget, 1);
649 		stb->seg_next = NULL;
650 		found_on_freelist = true;
651 		UVMHIST_LOG(pmapxtabhist, "freelist stb=%#jx",
652 		    (uintptr_t)stb, 0, 0, 0);
653 	}
654 	mutex_spin_exit(&pmap_segtab_lock);
655 
656 	struct vm_page *stb_pg = NULL;
657 	if (__predict_false(stb == NULL)) {
658 		stb_pg = pmap_pte_pagealloc();
659 
660 		if (__predict_false(stb_pg == NULL)) {
661 			/*
662 			 * XXX What else can we do?  Could we deadlock here?
663 			 */
664 			uvm_wait("segtab");
665 			goto again;
666 		}
667 		SEGTAB_ADD(npage, 1);
668 		const paddr_t stb_pa = VM_PAGE_TO_PHYS(stb_pg);
669 
670 		stb = (pmap_segtab_t *)PMAP_MAP_SEGTABPAGE(stb_pa);
671 		UVMHIST_LOG(pmapxtabhist, "new stb=%#jx", (uintptr_t)stb, 0,
672 		    0, 0);
673 #if 0
674 CTASSERT(NBPG / sizeof(*stb) == 1);
675 		const size_t n = NBPG / sizeof(*stb);
676 		if (n > 1) {
677 			/*
678 			 * link all the segtabs in this page together
679 			 */
680 			for (size_t i = 1; i < n - 1; i++) {
681 				stb[i].seg_next = &stb[i + 1];
682 			}
683 			/*
684 			 * Now link the new segtabs into the free segtab list.
685 			 */
686 			mutex_spin_enter(&pmap_segtab_lock);
687 			stb[n - 1].seg_next = pmap_segtab_info.segalloc.free_segtab;
688 			pmap_segtab_info.segalloc.free_segtab = stb + 1;
689 			SEGTAB_ADD(nput, n - 1);
690 			mutex_spin_exit(&pmap_segtab_lock);
691 		}
692 #endif
693 	}
694 
695 	UVMHIST_LOG(pmapxtabhist, "about to attach",  0, 0, 0, 0);
696 	pmap_page_attach(pmap, (vaddr_t)stb, stb_pg, &pmap->pm_segtab_list, 0);
697 
698 	pmap_check_stb(stb, __func__,
699 	    found_on_freelist ? "from free list" : "allocated");
700 
701 	UVMHIST_LOG(pmapxtabhist, "... stb %#jx found on freelist %zu",
702 	    (uintptr_t)stb, found_on_freelist, 0, 0);
703 
704 	return stb;
705 }
706 #endif
707 
708 #if defined(PMAP_HWPAGEWALKER)
709 static void
710 pmap_pdetab_free(pmap_pdetab_t *ptb)
711 {
712 	UVMHIST_FUNC(__func__);
713 	UVMHIST_CALLARGS(pmaphist, "ptb %#jx", (uintptr_t)ptb, 0, 0, 0);
714 	/*
715 	 * Insert the pdetab into the pdetab freelist.
716 	 */
717 	mutex_spin_enter(&pmap_segtab_lock);
718 	ptb->pde_next = pmap_segtab_info.pdealloc.free_pdetab;
719 	pmap_segtab_info.pdealloc.free_pdetab = ptb;
720 	PDETAB_ADD(nput, 1);
721 	mutex_spin_exit(&pmap_segtab_lock);
722 
723 }
724 #endif
725 
726 
727 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE)
728 /*
729  * Insert the segtab into the segtab freelist.
730  */
731 static void
732 pmap_segtab_free(pmap_segtab_t *stb)
733 {
734 	UVMHIST_FUNC(__func__);
735 	UVMHIST_CALLARGS(pmaphist, "stb %#jx", (uintptr_t)stb, 0, 0, 0);
736 
737 	/*
738 	 * Insert the segtab into the segtab freelist.
739 	 */
740 	mutex_spin_enter(&pmap_segtab_lock);
741 	stb->seg_next = pmap_segtab_info.segalloc.free_segtab;
742 	pmap_segtab_info.segalloc.free_segtab = stb;
743 	SEGTAB_ADD(nput, 1);
744 	mutex_spin_exit(&pmap_segtab_lock);
745 }
746 #endif
747 
748 #if defined(PMAP_HWPAGEWALKER)
749 static void
750 pmap_pdetab_release(pmap_t pmap, pmap_pdetab_t **ptb_p, bool free_ptb,
751     vaddr_t va, vsize_t vinc)
752 {
753 	const vaddr_t pdetab_mask = PMAP_PDETABSIZE - 1;
754 	pmap_pdetab_t *ptb = *ptb_p;
755 
756 	UVMHIST_FUNC(__func__);
757 	UVMHIST_CALLARGS(pmapxtabhist, "pm %#jx ptb_p %#jx ptb %#jx free %jd",
758 	    (uintptr_t)pmap, (uintptr_t)ptb_p, (uintptr_t)ptb, free_ptb);
759 	UVMHIST_LOG(pmapxtabhist, " va=%#jx vinc=%#jx",
760 	    (uintptr_t)va, (uintptr_t)vinc, 0, 0);
761 
762 	for (size_t i = (va / vinc) & pdetab_mask;
763 	    i < PMAP_PDETABSIZE;
764 	    i++, va += vinc) {
765 #ifdef _LP64
766 		if (vinc > NBSEG) {
767 			if (pte_pde_valid_p(ptb->pde_pde[i])) {
768 				pmap_pdetab_t *nptb =
769 				    pmap_pde_to_pdetab(ptb->pde_pde[i]);
770 				UVMHIST_LOG(pmapxtabhist,
771 				    " va %#jx ptp->pde_pde[%jd] (*%#jx) = %#jx "
772 				    "recursing", va, i, &ptb->pde_pde[i],
773 				    ptb->pde_pde[i]);
774 				pmap_pdetab_release(pmap, &nptb, true,
775 				    va, vinc / NPDEPG);
776 				ptb->pde_pde[i] = pte_invalid_pde();
777 				KASSERT(nptb == NULL);
778 			}
779 			continue;
780 		}
781 #endif
782 		KASSERT(vinc == NBSEG);
783 
784 		/* get pointer to PT page */
785 		pmap_ptpage_t *ppg = pmap_pde_to_ptpage(ptb->pde_pde[i]);
786 		UVMHIST_LOG(pmapxtabhist,
787 		    "   va %#jx ptb->pde_pde[%jd] (*%#jx) = %#jx", va, i,
788 		    (uintptr_t)&ptb->pde_pde[i], ptb->pde_pde[i]);
789 		if (ppg == NULL)
790 			continue;
791 
792 		UVMHIST_LOG(pmapxtabhist, " zeroing tab (%#jx)[%jd] (%#jx)",
793 		    (uintptr_t)ptb->pde_pde, i, (uintptr_t)&ptb->pde_pde[i], 0);
794 
795 		ptb->pde_pde[i] = pte_invalid_pde();
796 
797 		pmap_ptpage_free(pmap, ppg, __func__);
798 	}
799 
800 	if (free_ptb) {
801 		UVMHIST_LOG(pmapxtabhist, " ptbp %#jx ptb %#jx",
802 		    (uintptr_t)ptb_p, (uintptr_t)ptb, 0, 0);
803 		const vaddr_t kva = (vaddr_t)ptb;
804 		UVMHIST_LOG(pmapxtabhist, "about to detach",  0, 0, 0, 0);
805 		pmap_page_detach(pmap, &pmap->pm_pdetab_list, kva);
806 		pmap_pdetab_free(ptb);
807 		*ptb_p = NULL;
808 	}
809 }
810 #endif
811 
812 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE)
813 static void
814 pmap_segtab_release(pmap_t pmap, pmap_segtab_t **stb_p, bool free_stb,
815     pte_callback_t callback, uintptr_t flags, vaddr_t va, vsize_t vinc)
816 {
817 	pmap_segtab_t *stb = *stb_p;
818 
819 	UVMHIST_FUNC(__func__);
820 	UVMHIST_CALLARGS(pmapxtabhist, "pm=%#jx stb_p=%#jx free=%jd",
821 	    (uintptr_t)pmap, (uintptr_t)stb, free_stb, 0);
822 	UVMHIST_LOG(pmapxtabhist, " callback=%#jx flags=%#jx va=%#jx vinc=%#jx",
823 	    (uintptr_t)callback, flags, (uintptr_t)va, (uintptr_t)vinc);
824 
825 	for (size_t i = (va / vinc) & (PMAP_SEGTABSIZE - 1);
826 	    i < PMAP_SEGTABSIZE;
827 	    i++, va += vinc) {
828 #ifdef _LP64
829 		if (vinc > NBSEG) {
830 			if (stb->seg_seg[i] != NULL) {
831 				UVMHIST_LOG(pmapxtabhist,
832 				    " recursing %jd", i, 0, 0, 0);
833 				pmap_segtab_release(pmap, &stb->seg_seg[i],
834 				    true, callback, flags, va, vinc / NSEGPG);
835 				KASSERT(stb->seg_seg[i] == NULL);
836 			}
837 			continue;
838 		}
839 #endif
840 		KASSERT(vinc == NBSEG);
841 
842 		/* get pointer to segment map */
843 		pmap_ptpage_t *ppg = stb->seg_ppg[i];
844 		if (ppg == NULL)
845 			continue;
846 
847 		/*
848 		 * If our caller wants a callback, do so.
849 		 */
850 		if (callback != NULL) {
851 			(*callback)(pmap, va, va + vinc, ppg->ppg_ptes, flags);
852 		}
853 		pmap_ptpage_free(pmap, ppg, __func__);
854 		stb->seg_ppg[i] = NULL;
855 		UVMHIST_LOG(pmapxtabhist, " zeroing tab[%jd]", i, 0, 0, 0);
856 	}
857 
858 	if (free_stb) {
859 		pmap_check_stb(stb, __func__,
860 		    vinc == NBSEG ? "release seg" : "release xseg");
861 
862 		const vaddr_t kva = (vaddr_t)stb;
863 		UVMHIST_LOG(pmapxtabhist, "about to detach",  0, 0, 0, 0);
864 		pmap_page_detach(pmap, &pmap->pm_segtab_list, kva);
865 		pmap_segtab_free(stb);
866 		*stb_p = NULL;
867 	}
868 }
869 #endif
870 
871 
872 
873 /*
874  * Allocate the top segment table for the pmap.
875  */
876 void
877 pmap_segtab_init(pmap_t pmap)
878 {
879 	UVMHIST_FUNC(__func__);
880 	UVMHIST_CALLARGS(pmaphist, "pm %#jx", (uintptr_t)pmap, 0, 0, 0);
881 
882 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE)
883 	/*
884 	 * Constantly converting from extracted PA to VA is somewhat expensive
885 	 * for systems with hardware page walkers and without an inexpensive
886 	 * way to access arbitrary virtual addresses, so we allocate an extra
887 	 * root segtab so that it can contain non-virtual addresses.
888 	 */
889 	pmap->pm_segtab = pmap_segtab_alloc(pmap);
890 #endif
891 #if defined(PMAP_HWPAGEWALKER)
892 	pmap->pm_pdetab = pmap_pdetab_alloc(pmap);
893 	pmap_md_pdetab_init(pmap);
894 #endif
895 }
896 
897 /*
898  *	Retire the given physical map from service.
899  *	Should only be called if the map contains
900  *	no valid mappings.
901  */
902 void
903 pmap_segtab_destroy(pmap_t pmap, pte_callback_t func, uintptr_t flags)
904 {
905 	KASSERT(pmap != pmap_kernel());
906 #ifdef _LP64
907 	const vsize_t vinc = NBXSEG;
908 #else
909 	const vsize_t vinc = NBSEG;
910 #endif
911 
912 #if defined(PMAP_HWPAGEWALKER)
913 	if (pmap->pm_pdetab != NULL) {
914 		pmap_md_pdetab_fini(pmap);
915 		pmap_pdetab_release(pmap, &pmap->pm_pdetab,
916 		    true, pmap->pm_minaddr, vinc);
917 	}
918 #endif
919 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE)
920 	if (pmap->pm_segtab != NULL) {
921 		pmap_segtab_release(pmap, &pmap->pm_segtab,
922 		    func == NULL, func, flags, pmap->pm_minaddr, vinc);
923 	}
924 #endif
925 
926 #if defined(PMAP_HWPAGEWALKER)
927 #if !defined(PMAP_MAP_PDETABPAGE)
928 	KASSERT((pmap->pm_segtab == NULL) == (pmap->pm_pdetab == NULL));
929 #endif
930 	KASSERT(pmap->pm_pdetab == NULL);
931 #endif
932 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE)
933 	KASSERT(pmap->pm_segtab == NULL);
934 #endif
935 
936 }
937 
938 /*
939  *	Make a new pmap (vmspace) active for the given process.
940  */
941 void
942 pmap_segtab_activate(struct pmap *pm, struct lwp *l)
943 {
944 	if (l == curlwp) {
945 		KASSERT(pm == l->l_proc->p_vmspace->vm_map.pmap);
946 		pmap_md_xtab_activate(pm, l);
947 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE)
948 		struct cpu_info * const ci = l->l_cpu;
949 		if (pm == pmap_kernel()) {
950 			ci->ci_pmap_user_segtab = PMAP_INVALID_SEGTAB_ADDRESS;
951 #ifdef _LP64
952 			ci->ci_pmap_user_seg0tab = PMAP_INVALID_SEGTAB_ADDRESS;
953 #endif
954 		} else {
955 			ci->ci_pmap_user_segtab = pm->pm_segtab;
956 #ifdef _LP64
957 			ci->ci_pmap_user_seg0tab = pm->pm_segtab->seg_seg[0];
958 #endif
959 		}
960 #endif
961 	}
962 }
963 
964 void
965 pmap_segtab_deactivate(pmap_t pm)
966 {
967 	pmap_md_xtab_deactivate(pm);
968 
969 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE)
970 	curcpu()->ci_pmap_user_segtab = PMAP_INVALID_SEGTAB_ADDRESS;
971 #ifdef _LP64
972 	curcpu()->ci_pmap_user_seg0tab = NULL;
973 #endif
974 #endif
975 }
976 
977 /*
978  *	Act on the given range of addresses from the specified map.
979  *
980  *	It is assumed that the start and end are properly rounded to
981  *	the page size.
982  */
983 void
984 pmap_pte_process(pmap_t pmap, vaddr_t sva, vaddr_t eva,
985     pte_callback_t callback, uintptr_t flags)
986 {
987 #if 0
988 	printf("%s: %p, %"PRIxVADDR", %"PRIxVADDR", %p, %"PRIxPTR"\n",
989 	    __func__, pmap, sva, eva, callback, flags);
990 #endif
991 	while (sva < eva) {
992 		vaddr_t lastseg_va = pmap_trunc_seg(sva) + NBSEG;
993 		if (lastseg_va == 0 || lastseg_va > eva)
994 			lastseg_va = eva;
995 
996 		/*
997 		 * If VA belongs to an unallocated segment,
998 		 * skip to the next segment boundary.
999 		 */
1000 		pt_entry_t * const ptep = pmap_pte_lookup(pmap, sva);
1001 		if (ptep != NULL) {
1002 			/*
1003 			 * Callback to deal with the ptes for this segment.
1004 			 */
1005 			(*callback)(pmap, sva, lastseg_va, ptep, flags);
1006 		}
1007 		/*
1008 		 * In theory we could release pages with no entries,
1009 		 * but that takes more effort than we want here.
1010 		 */
1011 		sva = lastseg_va;
1012 	}
1013 }
1014 
1015 #if defined(PMAP_HWPAGEWALKER) && defined(PMAP_MAP_PDETABPAGE)
1016 static pd_entry_t *
1017 pmap_pdetab_reserve(struct pmap *pmap, vaddr_t va)
1018 #elif defined(PMAP_HWPAGEWALKER)
1019 static pmap_ptpage_t **
1020 pmap_segtab_reserve(struct pmap *pmap, vaddr_t va, pd_entry_t **pde_p)
1021 #else
1022 static pmap_ptpage_t **
1023 pmap_segtab_reserve(struct pmap *pmap, vaddr_t va)
1024 #endif
1025 {
1026 	UVMHIST_FUNC(__func__);
1027 	UVMHIST_CALLARGS(pmaphist, "pm %#jx va %#jx", (uintptr_t)pmap,
1028 	    (uintptr_t)va, 0, 0);
1029 
1030 #if defined(PMAP_HWPAGEWALKER)
1031 	pmap_pdetab_t *ptb = pmap->pm_pdetab;
1032 	UVMHIST_LOG(pmaphist, "pm_pdetab %#jx", (uintptr_t)ptb, 0, 0, 0);
1033 #endif
1034 #if defined(PMAP_HWPAGEWALKER) && defined(PMAP_MAP_PDETABPAGE)
1035 	vaddr_t segtab_mask = PMAP_PDETABSIZE - 1;
1036 #ifdef _LP64
1037 	for (size_t segshift = XSEGSHIFT;
1038 	    segshift > SEGSHIFT;
1039 	    segshift -= PGSHIFT - 3, segtab_mask = NSEGPG - 1) {
1040 		pd_entry_t * const pde_p =
1041 		    &ptb->pde_pde[(va >> segshift) & segtab_mask];
1042 		pd_entry_t opde = *pde_p;
1043 
1044 		UVMHIST_LOG(pmaphist,
1045 		    "ptb %#jx segshift %jd pde_p %#jx opde %#jx",
1046 		    ptb, segshift, pde_p, opde);
1047 
1048 		if (__predict_false(!pte_pde_valid_p(opde))) {
1049 			ptb = pmap_pdetab_alloc(pmap);
1050 			pd_entry_t npde = pte_pde_pdetab(
1051 			    pmap_md_direct_mapped_vaddr_to_paddr((vaddr_t)ptb),
1052 			    pmap == pmap_kernel());
1053 			opde = pte_pde_cas(pde_p, opde, npde);
1054 			if (__predict_false(pte_pde_valid_p(opde))) {
1055 				const vaddr_t kva = (vaddr_t)ptb;
1056 				UVMHIST_LOG(pmapxtabhist, "about to detach",
1057 				    0, 0, 0, 0);
1058 				pmap_page_detach(pmap, &pmap->pm_pdetab_list,
1059 				    kva);
1060 				pmap_pdetab_free(ptb);
1061 			} else {
1062 				opde = npde;
1063 			}
1064 		}
1065 		ptb = pmap_pde_to_pdetab(opde);
1066 		UVMHIST_LOG(pmaphist, "opde %#jx ptb %#jx", opde, ptb, 0, 0);
1067 	}
1068 #elif defined(XSEGSHIFT)
1069 	size_t segshift = XSEGSHIFT;
1070 
1071 	pd_entry_t opde = ptb->pde_pde[(va >> segshift) & segtab_mask];
1072 	KASSERT(pte_pde_valid_p(opde));
1073 	ptb = pmap_pde_to_pdetab(opde);
1074 	segtab_mask = NSEGPG - 1;
1075 #endif /* _LP64 */
1076 	const size_t idx = (va >> SEGSHIFT) & segtab_mask;
1077 
1078 	UVMHIST_LOG(pmaphist, "... returning %#jx (idx %jd)", (uintptr_t)&ptb->pde_pde[idx], idx, 0, 0);
1079 
1080 	return &ptb->pde_pde[idx];
1081 #else /* PMAP_HWPAGEWALKER && PMAP_MAP_PDETABPAGE */
1082 	pmap_segtab_t *stb = pmap->pm_segtab;
1083 	vaddr_t segtab_mask = PMAP_SEGTABSIZE - 1;
1084 #ifdef _LP64
1085 	for (size_t segshift = XSEGSHIFT;
1086 	    segshift > SEGSHIFT;
1087 	    segshift -= PGSHIFT - 3, segtab_mask = NSEGPG - 1) {
1088 		size_t idx = (va >> segshift) & segtab_mask;
1089 		pmap_segtab_t ** const stb_p = &stb->seg_seg[idx];
1090 #if defined(PMAP_HWPAGEWALKER)
1091 		pmap_pdetab_t ** const ptb_p = &ptb->pde_pde[idx];
1092 #endif	/* PMAP_HWPAGEWALKER */
1093 		if (__predict_false((stb = *stb_p) == NULL)) {
1094 			stb = pmap_segtab_alloc(pmap);
1095 #ifdef MULTIPROCESSOR
1096 			pmap_segtab_t *ostb = atomic_cas_ptr(stb_p, NULL, stb);
1097 			if (__predict_false(ostb != NULL)) {
1098 				const vaddr_t kva = (vaddr_t)stb;
1099 				UVMHIST_LOG(pmapxtabhist, "about to detach",
1100 				    0, 0, 0, 0);
1101 				pmap_page_detach(pmap, &pmap->pm_segtab_list,
1102 				    kva);
1103 				pmap_segtab_free(stb);
1104 				stb = ostb;
1105 			}
1106 #else
1107 			*stb_p = stb;
1108 #endif /* MULTIPROCESSOR */
1109 		}
1110 	}
1111 #elif defined(PMAP_HWPAGEWALKER)
1112 	pmap_segtab_t opde = ptb->pde_pde[(va >> segshift) & segtab_mask];
1113 	KASSERT(pte_pde_valid_p(opde));
1114 	ptb = pmap_pde_to_pdetab(opde);
1115 	segtab_mask = NSEGPG - 1;
1116 
1117 #endif /* _LP64 */
1118 	size_t idx = (va >> SEGSHIFT) & segtab_mask;
1119 #if defined(PMAP_HWPAGEWALKER)
1120 #if defined(XSEGSHIFT) && (XSEGSHIFT != SEGSHIFT)
1121 	*pte_p = &pmap->pm_segtab
1122 #else /* XSEGSHIFT */
1123 	*pde_p = &ptb->pde_pde[idx];
1124 #endif /* XSEGSHIFT */
1125 #endif /* PMAP_HWPAGEWALKER */
1126 	return &stb->seg_ppg[idx];
1127 #endif
1128 }
1129 
1130 
1131 /*
1132  *	Return a pointer for the pte that corresponds to the specified virtual
1133  *	address (va) in the target physical map, allocating if needed.
1134  */
1135 pt_entry_t *
1136 pmap_pte_reserve(pmap_t pmap, vaddr_t va, int flags)
1137 {
1138 	UVMHIST_FUNC(__func__);
1139 	UVMHIST_CALLARGS(pmaphist, "pm=%#jx va=%#jx flags=%#jx",
1140 	    (uintptr_t)pmap, (uintptr_t)va, flags, 0);
1141 	pmap_ptpage_t *ppg;
1142 	paddr_t pa = 0;
1143 
1144 #if defined(PMAP_HWPAGEWALKER) && defined(PMAP_MAP_PDETABPAGE)
1145 	pd_entry_t * const pde_p = pmap_pdetab_reserve(pmap, va);
1146 	ppg = pmap_pde_to_ptpage(*pde_p);
1147 #elif defined(PMAP_HWPAGEWALKER)
1148 	pd_entry_t *pde_p;
1149 	pmap_ptpage_t ** const ppg_p = pmap_segtab_reserve(pmap, va, &pde_p);
1150 	ppg = *ppg_p;
1151 #else
1152 	pmap_ptpage_t ** const ppg_p = pmap_segtab_reserve(pmap, va);
1153 	ppg = *ppg_p;
1154 #endif
1155 
1156 	if (__predict_false(ppg == NULL)) {
1157 		ppg = pmap_ptpage_alloc(pmap, flags, &pa);
1158 		if (__predict_false(ppg == NULL))
1159 			return NULL;
1160 
1161 #if defined(PMAP_HWPAGEWALKER)
1162 		pd_entry_t npde = pte_pde_ptpage(pa, pmap == pmap_kernel());
1163 #endif
1164 #if defined(PMAP_HWPAGEWALKER) && defined(PMAP_MAP_PDETABPAGE)
1165 		pd_entry_t opde = pte_pde_cas(pde_p, pte_invalid_pde(), npde);
1166 		if (__predict_false(pte_pde_valid_p(opde))) {
1167 			pmap_ptpage_free(pmap, ppg, __func__);
1168 			ppg = pmap_pde_to_ptpage(opde);
1169 		}
1170 #else
1171 #ifdef MULTIPROCESSOR
1172 		pmap_ptpage_t *oppg = atomic_cas_ptr(ppg_p, NULL, ppg);
1173 		/*
1174 		 * If another thread allocated the segtab needed for this va
1175 		 * free the page we just allocated.
1176 		 */
1177 		if (__predict_false(oppg != NULL)) {
1178 			pmap_ptpage_free(pmap, ppg, __func__);
1179 			ppg = oppg;
1180 #if defined(PMAP_HWPAGEWALKER)
1181 		} else {
1182 			pte_pde_set(pde_p, npde);
1183 #endif
1184 		}
1185 #else /* !MULTIPROCESSOR */
1186 		*ppg_p = ppg;
1187 #endif /* MULTIPROCESSOR */
1188 #endif /* PMAP_HWPAGEWALKER && PMAP_MAP_PDETABPAGE */
1189 	}
1190 
1191 	const size_t pte_idx = (va >> PGSHIFT) & (NPTEPG - 1);
1192 
1193 	return ppg->ppg_ptes + pte_idx;
1194 }
1195