xref: /netbsd-src/sys/uvm/pmap/pmap.c (revision d909946ca08dceb44d7d0f22ec9488679695d976)
1 /*	$NetBSD: pmap.c,v 1.20 2016/08/18 21:42:27 matt Exp $	*/
2 
3 /*-
4  * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9  * NASA Ames Research Center and by Chris G. Demetriou.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30  * POSSIBILITY OF SUCH DAMAGE.
31  */
32 
33 /*
34  * Copyright (c) 1992, 1993
35  *	The Regents of the University of California.  All rights reserved.
36  *
37  * This code is derived from software contributed to Berkeley by
38  * the Systems Programming Group of the University of Utah Computer
39  * Science Department and Ralph Campbell.
40  *
41  * Redistribution and use in source and binary forms, with or without
42  * modification, are permitted provided that the following conditions
43  * are met:
44  * 1. Redistributions of source code must retain the above copyright
45  *    notice, this list of conditions and the following disclaimer.
46  * 2. Redistributions in binary form must reproduce the above copyright
47  *    notice, this list of conditions and the following disclaimer in the
48  *    documentation and/or other materials provided with the distribution.
49  * 3. Neither the name of the University nor the names of its contributors
50  *    may be used to endorse or promote products derived from this software
51  *    without specific prior written permission.
52  *
53  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
54  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
55  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
56  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
57  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
58  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
59  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
60  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
61  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
62  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
63  * SUCH DAMAGE.
64  *
65  *	@(#)pmap.c	8.4 (Berkeley) 1/26/94
66  */
67 
68 #include <sys/cdefs.h>
69 
70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.20 2016/08/18 21:42:27 matt Exp $");
71 
72 /*
73  *	Manages physical address maps.
74  *
75  *	In addition to hardware address maps, this
76  *	module is called upon to provide software-use-only
77  *	maps which may or may not be stored in the same
78  *	form as hardware maps.  These pseudo-maps are
79  *	used to store intermediate results from copy
80  *	operations to and from address spaces.
81  *
82  *	Since the information managed by this module is
83  *	also stored by the logical address mapping module,
84  *	this module may throw away valid virtual-to-physical
85  *	mappings at almost any time.  However, invalidations
86  *	of virtual-to-physical mappings must be done as
87  *	requested.
88  *
89  *	In order to cope with hardware architectures which
90  *	make virtual-to-physical map invalidates expensive,
91  *	this module may delay invalidate or reduced protection
92  *	operations until such time as they are actually
93  *	necessary.  This module is given full information as
94  *	to which processors are currently using which maps,
95  *	and to when physical maps must be made correct.
96  */
97 
98 #include "opt_modular.h"
99 #include "opt_multiprocessor.h"
100 #include "opt_sysv.h"
101 
102 #define __PMAP_PRIVATE
103 
104 #include <sys/param.h>
105 #include <sys/atomic.h>
106 #include <sys/buf.h>
107 #include <sys/cpu.h>
108 #include <sys/mutex.h>
109 #include <sys/pool.h>
110 #include <sys/atomic.h>
111 #include <sys/mutex.h>
112 #include <sys/atomic.h>
113 
114 #include <uvm/uvm.h>
115 
116 #if defined(MULTIPROCESSOR) && defined(PMAP_VIRTUAL_CACHE_ALIASES) \
117     && !defined(PMAP_NO_PV_UNCACHED)
118 #error PMAP_VIRTUAL_CACHE_ALIASES with MULTIPROCESSOR requires \
119  PMAP_NO_PV_UNCACHED to be defined
120 #endif
121 
122 PMAP_COUNTER(remove_kernel_calls, "remove kernel calls");
123 PMAP_COUNTER(remove_kernel_pages, "kernel pages unmapped");
124 PMAP_COUNTER(remove_user_calls, "remove user calls");
125 PMAP_COUNTER(remove_user_pages, "user pages unmapped");
126 PMAP_COUNTER(remove_flushes, "remove cache flushes");
127 PMAP_COUNTER(remove_tlb_ops, "remove tlb ops");
128 PMAP_COUNTER(remove_pvfirst, "remove pv first");
129 PMAP_COUNTER(remove_pvsearch, "remove pv search");
130 
131 PMAP_COUNTER(prefer_requests, "prefer requests");
132 PMAP_COUNTER(prefer_adjustments, "prefer adjustments");
133 
134 PMAP_COUNTER(idlezeroed_pages, "pages idle zeroed");
135 
136 PMAP_COUNTER(kenter_pa, "kernel fast mapped pages");
137 PMAP_COUNTER(kenter_pa_bad, "kernel fast mapped pages (bad color)");
138 PMAP_COUNTER(kenter_pa_unmanaged, "kernel fast mapped unmanaged pages");
139 PMAP_COUNTER(kremove_pages, "kernel fast unmapped pages");
140 
141 PMAP_COUNTER(page_cache_evictions, "pages changed to uncacheable");
142 PMAP_COUNTER(page_cache_restorations, "pages changed to cacheable");
143 
144 PMAP_COUNTER(kernel_mappings_bad, "kernel pages mapped (bad color)");
145 PMAP_COUNTER(user_mappings_bad, "user pages mapped (bad color)");
146 PMAP_COUNTER(kernel_mappings, "kernel pages mapped");
147 PMAP_COUNTER(user_mappings, "user pages mapped");
148 PMAP_COUNTER(user_mappings_changed, "user mapping changed");
149 PMAP_COUNTER(kernel_mappings_changed, "kernel mapping changed");
150 PMAP_COUNTER(uncached_mappings, "uncached pages mapped");
151 PMAP_COUNTER(unmanaged_mappings, "unmanaged pages mapped");
152 PMAP_COUNTER(managed_mappings, "managed pages mapped");
153 PMAP_COUNTER(mappings, "pages mapped");
154 PMAP_COUNTER(remappings, "pages remapped");
155 PMAP_COUNTER(unmappings, "pages unmapped");
156 PMAP_COUNTER(primary_mappings, "page initial mappings");
157 PMAP_COUNTER(primary_unmappings, "page final unmappings");
158 PMAP_COUNTER(tlb_hit, "page mapping");
159 
160 PMAP_COUNTER(exec_mappings, "exec pages mapped");
161 PMAP_COUNTER(exec_synced_mappings, "exec pages synced");
162 PMAP_COUNTER(exec_synced_remove, "exec pages synced (PR)");
163 PMAP_COUNTER(exec_synced_clear_modify, "exec pages synced (CM)");
164 PMAP_COUNTER(exec_synced_page_protect, "exec pages synced (PP)");
165 PMAP_COUNTER(exec_synced_protect, "exec pages synced (P)");
166 PMAP_COUNTER(exec_uncached_page_protect, "exec pages uncached (PP)");
167 PMAP_COUNTER(exec_uncached_clear_modify, "exec pages uncached (CM)");
168 PMAP_COUNTER(exec_uncached_zero_page, "exec pages uncached (ZP)");
169 PMAP_COUNTER(exec_uncached_copy_page, "exec pages uncached (CP)");
170 PMAP_COUNTER(exec_uncached_remove, "exec pages uncached (PR)");
171 
172 PMAP_COUNTER(create, "creates");
173 PMAP_COUNTER(reference, "references");
174 PMAP_COUNTER(dereference, "dereferences");
175 PMAP_COUNTER(destroy, "destroyed");
176 PMAP_COUNTER(activate, "activations");
177 PMAP_COUNTER(deactivate, "deactivations");
178 PMAP_COUNTER(update, "updates");
179 #ifdef MULTIPROCESSOR
180 PMAP_COUNTER(shootdown_ipis, "shootdown IPIs");
181 #endif
182 PMAP_COUNTER(unwire, "unwires");
183 PMAP_COUNTER(copy, "copies");
184 PMAP_COUNTER(clear_modify, "clear_modifies");
185 PMAP_COUNTER(protect, "protects");
186 PMAP_COUNTER(page_protect, "page_protects");
187 
188 #define PMAP_ASID_RESERVED 0
189 CTASSERT(PMAP_ASID_RESERVED == 0);
190 
191 #ifndef PMAP_SEGTAB_ALIGN
192 #define PMAP_SEGTAB_ALIGN	/* nothing */
193 #endif
194 #ifdef _LP64
195 pmap_segtab_t	pmap_kstart_segtab PMAP_SEGTAB_ALIGN; /* first mid-level segtab for kernel */
196 #endif
197 pmap_segtab_t	pmap_kern_segtab PMAP_SEGTAB_ALIGN = { /* top level segtab for kernel */
198 #ifdef _LP64
199 	.seg_seg[(VM_MIN_KERNEL_ADDRESS & XSEGOFSET) >> SEGSHIFT] = &pmap_kstart_segtab,
200 #endif
201 };
202 
203 struct pmap_kernel kernel_pmap_store = {
204 	.kernel_pmap = {
205 		.pm_count = 1,
206 		.pm_segtab = &pmap_kern_segtab,
207 		.pm_minaddr = VM_MIN_KERNEL_ADDRESS,
208 		.pm_maxaddr = VM_MAX_KERNEL_ADDRESS,
209 	},
210 };
211 
212 struct pmap * const kernel_pmap_ptr = &kernel_pmap_store.kernel_pmap;
213 
214 struct pmap_limits pmap_limits = {	/* VA and PA limits */
215 	.virtual_start = VM_MIN_KERNEL_ADDRESS,
216 };
217 
218 #ifdef UVMHIST
219 static struct kern_history_ent pmapexechistbuf[10000];
220 static struct kern_history_ent pmaphistbuf[10000];
221 UVMHIST_DEFINE(pmapexechist);
222 UVMHIST_DEFINE(pmaphist);
223 #endif
224 
225 /*
226  * The pools from which pmap structures and sub-structures are allocated.
227  */
228 struct pool pmap_pmap_pool;
229 struct pool pmap_pv_pool;
230 
231 #ifndef PMAP_PV_LOWAT
232 #define	PMAP_PV_LOWAT	16
233 #endif
234 int	pmap_pv_lowat = PMAP_PV_LOWAT;
235 
236 bool	pmap_initialized = false;
237 #define	PMAP_PAGE_COLOROK_P(a, b) \
238 		((((int)(a) ^ (int)(b)) & pmap_page_colormask) == 0)
239 u_int	pmap_page_colormask;
240 
241 #define PAGE_IS_MANAGED(pa)	(pmap_initialized && uvm_pageismanaged(pa))
242 
243 #define PMAP_IS_ACTIVE(pm)						\
244 	((pm) == pmap_kernel() || 					\
245 	 (pm) == curlwp->l_proc->p_vmspace->vm_map.pmap)
246 
247 /* Forward function declarations */
248 void pmap_page_remove(struct vm_page *);
249 static void pmap_pvlist_check(struct vm_page_md *);
250 void pmap_remove_pv(pmap_t, vaddr_t, struct vm_page *, bool);
251 void pmap_enter_pv(pmap_t, vaddr_t, struct vm_page *, pt_entry_t *, u_int);
252 
253 /*
254  * PV table management functions.
255  */
256 void	*pmap_pv_page_alloc(struct pool *, int);
257 void	pmap_pv_page_free(struct pool *, void *);
258 
259 struct pool_allocator pmap_pv_page_allocator = {
260 	pmap_pv_page_alloc, pmap_pv_page_free, 0,
261 };
262 
263 #define	pmap_pv_alloc()		pool_get(&pmap_pv_pool, PR_NOWAIT)
264 #define	pmap_pv_free(pv)	pool_put(&pmap_pv_pool, (pv))
265 
266 #if !defined(MULTIPROCESSOR) || !defined(PMAP_MD_NEED_TLB_MISS_LOCK)
267 #define	pmap_md_tlb_miss_lock_enter()	do { } while(/*CONSTCOND*/0)
268 #define	pmap_md_tlb_miss_lock_exit()	do { } while(/*CONSTCOND*/0)
269 #endif /* !MULTIPROCESSOR || !PMAP_MD_NEED_TLB_MISS_LOCK */
270 
271 #ifndef MULTIPROCESSOR
272 kmutex_t pmap_pvlist_mutex	__cacheline_aligned;
273 #endif
274 
275 /*
276  * Debug functions.
277  */
278 
279 #ifdef DEBUG
280 static inline void
281 pmap_asid_check(pmap_t pm, const char *func)
282 {
283 	if (!PMAP_IS_ACTIVE(pm))
284 		return;
285 
286 	struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(curcpu()));
287 	tlb_asid_t asid = tlb_get_asid();
288 	if (asid != pai->pai_asid)
289 		panic("%s: inconsistency for active TLB update: %u <-> %u",
290 		    func, asid, pai->pai_asid);
291 }
292 #endif
293 
294 static void
295 pmap_addr_range_check(pmap_t pmap, vaddr_t sva, vaddr_t eva, const char *func)
296 {
297 #ifdef DEBUG
298 	if (pmap == pmap_kernel()) {
299 		if (sva < VM_MIN_KERNEL_ADDRESS)
300 			panic("%s: kva %#"PRIxVADDR" not in range",
301 			    func, sva);
302 		if (eva >= pmap_limits.virtual_end)
303 			panic("%s: kva %#"PRIxVADDR" not in range",
304 			    func, eva);
305 	} else {
306 		if (eva > VM_MAXUSER_ADDRESS)
307 			panic("%s: uva %#"PRIxVADDR" not in range",
308 			    func, eva);
309 		pmap_asid_check(pmap, func);
310 	}
311 #endif
312 }
313 
314 /*
315  * Misc. functions.
316  */
317 
318 bool
319 pmap_page_clear_attributes(struct vm_page_md *mdpg, u_int clear_attributes)
320 {
321 	volatile unsigned long * const attrp = &mdpg->mdpg_attrs;
322 #ifdef MULTIPROCESSOR
323 	for (;;) {
324 		u_int old_attr = *attrp;
325 		if ((old_attr & clear_attributes) == 0)
326 			return false;
327 		u_int new_attr = old_attr & ~clear_attributes;
328 		if (old_attr == atomic_cas_ulong(attrp, old_attr, new_attr))
329 			return true;
330 	}
331 #else
332 	unsigned long old_attr = *attrp;
333 	if ((old_attr & clear_attributes) == 0)
334 		return false;
335 	*attrp &= ~clear_attributes;
336 	return true;
337 #endif
338 }
339 
340 void
341 pmap_page_set_attributes(struct vm_page_md *mdpg, u_int set_attributes)
342 {
343 #ifdef MULTIPROCESSOR
344 	atomic_or_ulong(&mdpg->mdpg_attrs, set_attributes);
345 #else
346 	mdpg->mdpg_attrs |= set_attributes;
347 #endif
348 }
349 
350 static void
351 pmap_page_syncicache(struct vm_page *pg)
352 {
353 #ifndef MULTIPROCESSOR
354 	struct pmap * const curpmap = curlwp->l_proc->p_vmspace->vm_map.pmap;
355 #endif
356 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
357 	pv_entry_t pv = &mdpg->mdpg_first;
358 	kcpuset_t *onproc;
359 #ifdef MULTIPROCESSOR
360 	kcpuset_create(&onproc, true);
361 	KASSERT(onproc != NULL);
362 #else
363 	onproc = NULL;
364 #endif
365 	VM_PAGEMD_PVLIST_READLOCK(mdpg);
366 	pmap_pvlist_check(mdpg);
367 
368 	if (pv->pv_pmap != NULL) {
369 		for (; pv != NULL; pv = pv->pv_next) {
370 #ifdef MULTIPROCESSOR
371 			kcpuset_merge(onproc, pv->pv_pmap->pm_onproc);
372 			if (kcpuset_match(onproc, kcpuset_running)) {
373 				break;
374 			}
375 #else
376 			if (pv->pv_pmap == curpmap) {
377 				onproc = curcpu()->ci_data.cpu_kcpuset;
378 				break;
379 			}
380 #endif
381 		}
382 	}
383 	pmap_pvlist_check(mdpg);
384 	VM_PAGEMD_PVLIST_UNLOCK(mdpg);
385 	kpreempt_disable();
386 	pmap_md_page_syncicache(pg, onproc);
387 	kpreempt_enable();
388 #ifdef MULTIPROCESSOR
389 	kcpuset_destroy(onproc);
390 #endif
391 }
392 
393 /*
394  * Define the initial bounds of the kernel virtual address space.
395  */
396 void
397 pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp)
398 {
399 
400 	*vstartp = pmap_limits.virtual_start;
401 	*vendp = pmap_limits.virtual_end;
402 }
403 
404 vaddr_t
405 pmap_growkernel(vaddr_t maxkvaddr)
406 {
407 	vaddr_t virtual_end = pmap_limits.virtual_end;
408 	maxkvaddr = pmap_round_seg(maxkvaddr) - 1;
409 
410 	/*
411 	 * Reserve PTEs for the new KVA space.
412 	 */
413 	for (; virtual_end < maxkvaddr; virtual_end += NBSEG) {
414 		pmap_pte_reserve(pmap_kernel(), virtual_end, 0);
415 	}
416 
417 	/*
418 	 * Don't exceed VM_MAX_KERNEL_ADDRESS!
419 	 */
420 	if (virtual_end == 0 || virtual_end > VM_MAX_KERNEL_ADDRESS)
421 		virtual_end = VM_MAX_KERNEL_ADDRESS;
422 
423 	/*
424 	 * Update new end.
425 	 */
426 	pmap_limits.virtual_end = virtual_end;
427 	return virtual_end;
428 }
429 
430 /*
431  * Bootstrap memory allocator (alternative to vm_bootstrap_steal_memory()).
432  * This function allows for early dynamic memory allocation until the virtual
433  * memory system has been bootstrapped.  After that point, either kmem_alloc
434  * or malloc should be used.  This function works by stealing pages from the
435  * (to be) managed page pool, then implicitly mapping the pages (by using
436  * their k0seg addresses) and zeroing them.
437  *
438  * It may be used once the physical memory segments have been pre-loaded
439  * into the vm_physmem[] array.  Early memory allocation MUST use this
440  * interface!  This cannot be used after vm_page_startup(), and will
441  * generate a panic if tried.
442  *
443  * Note that this memory will never be freed, and in essence it is wired
444  * down.
445  *
446  * We must adjust *vstartp and/or *vendp iff we use address space
447  * from the kernel virtual address range defined by pmap_virtual_space().
448  */
449 vaddr_t
450 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp)
451 {
452 	size_t npgs;
453 	paddr_t pa;
454 	vaddr_t va;
455 	struct vm_physseg *maybe_seg = NULL;
456 	u_int maybe_bank = vm_nphysseg;
457 
458 	size = round_page(size);
459 	npgs = atop(size);
460 
461 	aprint_debug("%s: need %zu pages\n", __func__, npgs);
462 
463 	for (u_int bank = 0; bank < vm_nphysseg; bank++) {
464 		struct vm_physseg * const seg = VM_PHYSMEM_PTR(bank);
465 		if (uvm.page_init_done == true)
466 			panic("pmap_steal_memory: called _after_ bootstrap");
467 
468 		aprint_debug("%s: seg %u: %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR"\n",
469 		    __func__, bank,
470 		    seg->avail_start, seg->start,
471 		    seg->avail_end, seg->end);
472 
473 		if (seg->avail_start != seg->start
474 		    || seg->avail_start >= seg->avail_end) {
475 			aprint_debug("%s: seg %u: bad start\n", __func__, bank);
476 			continue;
477 		}
478 
479 		if (seg->avail_end - seg->avail_start < npgs) {
480 			aprint_debug("%s: seg %u: too small for %zu pages\n",
481 			    __func__, bank, npgs);
482 			continue;
483 		}
484 
485 		if (!pmap_md_ok_to_steal_p(seg, npgs)) {
486 			continue;
487 		}
488 
489 		/*
490 		 * Always try to allocate from the segment with the least
491 		 * amount of space left.
492 		 */
493 #define VM_PHYSMEM_SPACE(s)	((s)->avail_end - (s)->avail_start)
494 		if (maybe_seg == NULL
495 		    || VM_PHYSMEM_SPACE(seg) < VM_PHYSMEM_SPACE(maybe_seg)) {
496 			maybe_seg = seg;
497 			maybe_bank = bank;
498 		}
499 	}
500 
501 	if (maybe_seg) {
502 		struct vm_physseg * const seg = maybe_seg;
503 		u_int bank = maybe_bank;
504 
505 		/*
506 		 * There are enough pages here; steal them!
507 		 */
508 		pa = ptoa(seg->avail_start);
509 		seg->avail_start += npgs;
510 		seg->start += npgs;
511 
512 		/*
513 		 * Have we used up this segment?
514 		 */
515 		if (seg->avail_start == seg->end) {
516 			if (vm_nphysseg == 1)
517 				panic("pmap_steal_memory: out of memory!");
518 
519 			aprint_debug("%s: seg %u: %zu pages stolen (removed)\n",
520 			    __func__, bank, npgs);
521 			/* Remove this segment from the list. */
522 			vm_nphysseg--;
523 			for (u_int x = bank; x < vm_nphysseg; x++) {
524 				/* structure copy */
525 				VM_PHYSMEM_PTR_SWAP(x, x + 1);
526 			}
527 		} else {
528 			aprint_debug("%s: seg %u: %zu pages stolen (%#"PRIxPADDR" left)\n",
529 			    __func__, bank, npgs, VM_PHYSMEM_SPACE(seg));
530 		}
531 
532 		va = pmap_md_map_poolpage(pa, size);
533 		memset((void *)va, 0, size);
534 		return va;
535 	}
536 
537 	/*
538 	 * If we got here, there was no memory left.
539 	 */
540 	panic("pmap_steal_memory: no memory to steal %zu pages", npgs);
541 }
542 
543 /*
544  *	Initialize the pmap module.
545  *	Called by vm_init, to initialize any structures that the pmap
546  *	system needs to map virtual memory.
547  */
548 void
549 pmap_init(void)
550 {
551 	UVMHIST_INIT_STATIC(pmapexechist, pmapexechistbuf);
552 	UVMHIST_INIT_STATIC(pmaphist, pmaphistbuf);
553 
554 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
555 
556 	/*
557 	 * Initialize the segtab lock.
558 	 */
559 	mutex_init(&pmap_segtab_lock, MUTEX_DEFAULT, IPL_HIGH);
560 
561 	/*
562 	 * Set a low water mark on the pv_entry pool, so that we are
563 	 * more likely to have these around even in extreme memory
564 	 * starvation.
565 	 */
566 	pool_setlowat(&pmap_pv_pool, pmap_pv_lowat);
567 
568 	/*
569 	 * Set the page colormask but allow pmap_md_init to override it.
570 	 */
571 	pmap_page_colormask = ptoa(uvmexp.colormask);
572 
573 	pmap_md_init();
574 
575 	/*
576 	 * Now it is safe to enable pv entry recording.
577 	 */
578 	pmap_initialized = true;
579 }
580 
581 /*
582  *	Create and return a physical map.
583  *
584  *	If the size specified for the map
585  *	is zero, the map is an actual physical
586  *	map, and may be referenced by the
587  *	hardware.
588  *
589  *	If the size specified is non-zero,
590  *	the map will be used in software only, and
591  *	is bounded by that size.
592  */
593 pmap_t
594 pmap_create(void)
595 {
596 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
597 	PMAP_COUNT(create);
598 
599 	pmap_t pmap = pool_get(&pmap_pmap_pool, PR_WAITOK);
600 	memset(pmap, 0, PMAP_SIZE);
601 
602 	KASSERT(pmap->pm_pai[0].pai_link.le_prev == NULL);
603 
604 	pmap->pm_count = 1;
605 	pmap->pm_minaddr = VM_MIN_ADDRESS;
606 	pmap->pm_maxaddr = VM_MAXUSER_ADDRESS;
607 
608 	pmap_segtab_init(pmap);
609 
610 #ifdef MULTIPROCESSOR
611 	kcpuset_create(&pmap->pm_active, true);
612 	kcpuset_create(&pmap->pm_onproc, true);
613 	KASSERT(pmap->pm_active != NULL);
614 	KASSERT(pmap->pm_onproc != NULL);
615 #endif
616 
617 	UVMHIST_LOG(pmaphist, " <-- done (pmap=%p)", pmap, 0, 0, 0);
618 
619 	return pmap;
620 }
621 
622 /*
623  *	Retire the given physical map from service.
624  *	Should only be called if the map contains
625  *	no valid mappings.
626  */
627 void
628 pmap_destroy(pmap_t pmap)
629 {
630 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
631 	UVMHIST_LOG(pmaphist, "(pmap=%p)", pmap, 0, 0, 0);
632 
633 	if (atomic_dec_uint_nv(&pmap->pm_count) > 0) {
634 		PMAP_COUNT(dereference);
635 		UVMHIST_LOG(pmaphist, " <-- done (deref)", 0, 0, 0, 0);
636 		return;
637 	}
638 
639 	PMAP_COUNT(destroy);
640 	KASSERT(pmap->pm_count == 0);
641 	kpreempt_disable();
642 	pmap_md_tlb_miss_lock_enter();
643 	pmap_tlb_asid_release_all(pmap);
644 	pmap_segtab_destroy(pmap, NULL, 0);
645 	pmap_md_tlb_miss_lock_exit();
646 
647 #ifdef MULTIPROCESSOR
648 	kcpuset_destroy(pmap->pm_active);
649 	kcpuset_destroy(pmap->pm_onproc);
650 	pmap->pm_active = NULL;
651 	pmap->pm_onproc = NULL;
652 #endif
653 
654 	pool_put(&pmap_pmap_pool, pmap);
655 	kpreempt_enable();
656 
657 	UVMHIST_LOG(pmaphist, " <-- done (freed)", 0, 0, 0, 0);
658 }
659 
660 /*
661  *	Add a reference to the specified pmap.
662  */
663 void
664 pmap_reference(pmap_t pmap)
665 {
666 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
667 	UVMHIST_LOG(pmaphist, "(pmap=%p)", pmap, 0, 0, 0);
668 	PMAP_COUNT(reference);
669 
670 	if (pmap != NULL) {
671 		atomic_inc_uint(&pmap->pm_count);
672 	}
673 
674 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
675 }
676 
677 /*
678  *	Make a new pmap (vmspace) active for the given process.
679  */
680 void
681 pmap_activate(struct lwp *l)
682 {
683 	pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap;
684 
685 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
686 	UVMHIST_LOG(pmaphist, "(l=%p (pmap=%p))", l, pmap, 0, 0);
687 	PMAP_COUNT(activate);
688 
689 	kpreempt_disable();
690 	pmap_md_tlb_miss_lock_enter();
691 	pmap_tlb_asid_acquire(pmap, l);
692 	if (l == curlwp) {
693 		pmap_segtab_activate(pmap, l);
694 	}
695 	pmap_md_tlb_miss_lock_exit();
696 	kpreempt_enable();
697 
698 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
699 }
700 
701 /*
702  * Remove this page from all physical maps in which it resides.
703  * Reflects back modify bits to the pager.
704  */
705 void
706 pmap_page_remove(struct vm_page *pg)
707 {
708 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
709 
710 	kpreempt_disable();
711 	VM_PAGEMD_PVLIST_LOCK(mdpg);
712 	pmap_pvlist_check(mdpg);
713 
714 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
715 
716 	pv_entry_t pv = &mdpg->mdpg_first;
717 	if (pv->pv_pmap == NULL) {
718 		VM_PAGEMD_PVLIST_UNLOCK(mdpg);
719 		kpreempt_enable();
720 		UVMHIST_LOG(pmaphist, " <-- done (empty)", 0, 0, 0, 0);
721 		return;
722 	}
723 
724 	pv_entry_t npv;
725 	pv_entry_t pvp = NULL;
726 
727 	for (; pv != NULL; pv = npv) {
728 		npv = pv->pv_next;
729 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
730 		if (pv->pv_va & PV_KENTER) {
731 			UVMHIST_LOG(pmaphist, " pv %p pmap %p va %"
732 			    PRIxVADDR" skip", pv, pv->pv_pmap, pv->pv_va, 0);
733 
734 			KASSERT(pv->pv_pmap == pmap_kernel());
735 
736 			/* Assume no more - it'll get fixed if there are */
737 			pv->pv_next = NULL;
738 
739 			/*
740 			 * pvp is non-null when we already have a PV_KENTER
741 			 * pv in pvh_first; otherwise we haven't seen a
742 			 * PV_KENTER pv and we need to copy this one to
743 			 * pvh_first
744 			 */
745 			if (pvp) {
746 				/*
747 				 * The previous PV_KENTER pv needs to point to
748 				 * this PV_KENTER pv
749 				 */
750 				pvp->pv_next = pv;
751 			} else {
752 				pv_entry_t fpv = &mdpg->mdpg_first;
753 				*fpv = *pv;
754 				KASSERT(fpv->pv_pmap == pmap_kernel());
755 			}
756 			pvp = pv;
757 			continue;
758 		}
759 #endif
760 		const pmap_t pmap = pv->pv_pmap;
761 		vaddr_t va = trunc_page(pv->pv_va);
762 		pt_entry_t * const ptep = pmap_pte_lookup(pmap, va);
763 		KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va,
764 		    pmap_limits.virtual_end);
765 		pt_entry_t pte = *ptep;
766 		UVMHIST_LOG(pmaphist, " pv %p pmap %p va %"PRIxVADDR
767 		    " pte %#"PRIxPTE, pv, pmap, va, pte_value(pte));
768 		if (!pte_valid_p(pte))
769 			continue;
770 		const bool is_kernel_pmap_p = (pmap == pmap_kernel());
771 		if (is_kernel_pmap_p) {
772 			PMAP_COUNT(remove_kernel_pages);
773 		} else {
774 			PMAP_COUNT(remove_user_pages);
775 		}
776 		if (pte_wired_p(pte))
777 			pmap->pm_stats.wired_count--;
778 		pmap->pm_stats.resident_count--;
779 
780 		pmap_md_tlb_miss_lock_enter();
781 		const pt_entry_t npte = pte_nv_entry(is_kernel_pmap_p);
782 		*ptep = npte;
783 		/*
784 		 * Flush the TLB for the given address.
785 		 */
786 		pmap_tlb_invalidate_addr(pmap, va);
787 		pmap_md_tlb_miss_lock_exit();
788 
789 		/*
790 		 * non-null means this is a non-pvh_first pv, so we should
791 		 * free it.
792 		 */
793 		if (pvp) {
794 			KASSERT(pvp->pv_pmap == pmap_kernel());
795 			KASSERT(pvp->pv_next == NULL);
796 			pmap_pv_free(pv);
797 		} else {
798 			pv->pv_pmap = NULL;
799 			pv->pv_next = NULL;
800 		}
801 	}
802 
803 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
804 	pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED);
805 #endif
806 	pmap_pvlist_check(mdpg);
807 	VM_PAGEMD_PVLIST_UNLOCK(mdpg);
808 	kpreempt_enable();
809 
810 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
811 }
812 
813 
814 /*
815  *	Make a previously active pmap (vmspace) inactive.
816  */
817 void
818 pmap_deactivate(struct lwp *l)
819 {
820 	pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap;
821 
822 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
823 	UVMHIST_LOG(pmaphist, "(l=%p (pmap=%p))", l, pmap, 0, 0);
824 	PMAP_COUNT(deactivate);
825 
826 	kpreempt_disable();
827 	KASSERT(l == curlwp || l->l_cpu == curlwp->l_cpu);
828 	pmap_md_tlb_miss_lock_enter();
829 	curcpu()->ci_pmap_user_segtab = PMAP_INVALID_SEGTAB_ADDRESS;
830 #ifdef _LP64
831 	curcpu()->ci_pmap_user_seg0tab = NULL;
832 #endif
833 	pmap_tlb_asid_deactivate(pmap);
834 	pmap_md_tlb_miss_lock_exit();
835 	kpreempt_enable();
836 
837 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
838 }
839 
840 void
841 pmap_update(struct pmap *pmap)
842 {
843 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
844 	UVMHIST_LOG(pmaphist, "(pmap=%p)", pmap, 0, 0, 0);
845 	PMAP_COUNT(update);
846 
847 	kpreempt_disable();
848 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN)
849 	u_int pending = atomic_swap_uint(&pmap->pm_shootdown_pending, 0);
850 	if (pending && pmap_tlb_shootdown_bystanders(pmap))
851 		PMAP_COUNT(shootdown_ipis);
852 #endif
853 	pmap_md_tlb_miss_lock_enter();
854 #if defined(DEBUG) && !defined(MULTIPROCESSOR)
855 	pmap_tlb_check(pmap, pmap_md_tlb_check_entry);
856 #endif /* DEBUG */
857 
858 	/*
859 	 * If pmap_remove_all was called, we deactivated ourselves and nuked
860 	 * our ASID.  Now we have to reactivate ourselves.
861 	 */
862 	if (__predict_false(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE)) {
863 		pmap->pm_flags ^= PMAP_DEFERRED_ACTIVATE;
864 		pmap_tlb_asid_acquire(pmap, curlwp);
865 		pmap_segtab_activate(pmap, curlwp);
866 	}
867 	pmap_md_tlb_miss_lock_exit();
868 	kpreempt_enable();
869 
870 	UVMHIST_LOG(pmaphist, " <-- done%s",
871 	    (pmap == pmap_kernel()) ? " (kernel)" : "", 0, 0, 0);
872 }
873 
874 /*
875  *	Remove the given range of addresses from the specified map.
876  *
877  *	It is assumed that the start and end are properly
878  *	rounded to the page size.
879  */
880 
881 static bool
882 pmap_pte_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep,
883 	uintptr_t flags)
884 {
885 	const pt_entry_t npte = flags;
886 	const bool is_kernel_pmap_p = (pmap == pmap_kernel());
887 
888 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
889 	UVMHIST_LOG(pmaphist, "(pmap=%p %sva=%#"PRIxVADDR"..%#"PRIxVADDR,
890 	    pmap, (is_kernel_pmap_p ? "(kernel) " : ""), sva, eva);
891 	UVMHIST_LOG(pmaphist, "ptep=%p, flags(npte)=%#"PRIxPTR")",
892 	    ptep, flags, 0, 0);
893 
894 	KASSERT(kpreempt_disabled());
895 
896 	for (; sva < eva; sva += NBPG, ptep++) {
897 		const pt_entry_t pte = *ptep;
898 		if (!pte_valid_p(pte))
899 			continue;
900 		if (is_kernel_pmap_p) {
901 			PMAP_COUNT(remove_kernel_pages);
902 		} else {
903 			PMAP_COUNT(remove_user_pages);
904 		}
905 		if (pte_wired_p(pte))
906 			pmap->pm_stats.wired_count--;
907 		pmap->pm_stats.resident_count--;
908 		struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte));
909 		if (__predict_true(pg != NULL)) {
910 			pmap_remove_pv(pmap, sva, pg, pte_modified_p(pte));
911 		}
912 		pmap_md_tlb_miss_lock_enter();
913 		*ptep = npte;
914 		/*
915 		 * Flush the TLB for the given address.
916 		 */
917 		pmap_tlb_invalidate_addr(pmap, sva);
918 		pmap_md_tlb_miss_lock_exit();
919 	}
920 
921 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
922 
923 	return false;
924 }
925 
926 void
927 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva)
928 {
929 	const bool is_kernel_pmap_p = (pmap == pmap_kernel());
930 	const pt_entry_t npte = pte_nv_entry(is_kernel_pmap_p);
931 
932 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
933 	UVMHIST_LOG(pmaphist, "(pmap=%p, va=%#"PRIxVADDR"..%#"PRIxVADDR")",
934 	    pmap, sva, eva, 0);
935 
936 	if (is_kernel_pmap_p) {
937 		PMAP_COUNT(remove_kernel_calls);
938 	} else {
939 		PMAP_COUNT(remove_user_calls);
940 	}
941 #ifdef PMAP_FAULTINFO
942 	curpcb->pcb_faultinfo.pfi_faultaddr = 0;
943 	curpcb->pcb_faultinfo.pfi_repeats = 0;
944 	curpcb->pcb_faultinfo.pfi_faultpte = NULL;
945 #endif
946 	kpreempt_disable();
947 	pmap_addr_range_check(pmap, sva, eva, __func__);
948 	pmap_pte_process(pmap, sva, eva, pmap_pte_remove, npte);
949 	kpreempt_enable();
950 
951 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
952 }
953 
954 /*
955  *	pmap_page_protect:
956  *
957  *	Lower the permission for all mappings to a given page.
958  */
959 void
960 pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
961 {
962 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
963 	pv_entry_t pv;
964 	vaddr_t va;
965 
966 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
967 	UVMHIST_LOG(pmaphist, "(pg=%p (pa %#"PRIxPADDR") prot=%#x)",
968 	    pg, VM_PAGE_TO_PHYS(pg), prot, 0);
969 	PMAP_COUNT(page_protect);
970 
971 	switch (prot) {
972 	case VM_PROT_READ|VM_PROT_WRITE:
973 	case VM_PROT_ALL:
974 		break;
975 
976 	/* copy_on_write */
977 	case VM_PROT_READ:
978 	case VM_PROT_READ|VM_PROT_EXECUTE:
979 		pv = &mdpg->mdpg_first;
980 		kpreempt_disable();
981 		VM_PAGEMD_PVLIST_READLOCK(mdpg);
982 		pmap_pvlist_check(mdpg);
983 		/*
984 		 * Loop over all current mappings setting/clearing as apropos.
985 		 */
986 		if (pv->pv_pmap != NULL) {
987 			while (pv != NULL) {
988 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
989 				if (pv->pv_va & PV_KENTER) {
990 					pv = pv->pv_next;
991 					continue;
992 				}
993 #endif
994 				const pmap_t pmap = pv->pv_pmap;
995 				va = trunc_page(pv->pv_va);
996 				const uintptr_t gen =
997 				    VM_PAGEMD_PVLIST_UNLOCK(mdpg);
998 				pmap_protect(pmap, va, va + PAGE_SIZE, prot);
999 				KASSERT(pv->pv_pmap == pmap);
1000 				pmap_update(pmap);
1001 				if (gen != VM_PAGEMD_PVLIST_READLOCK(mdpg)) {
1002 					pv = &mdpg->mdpg_first;
1003 				} else {
1004 					pv = pv->pv_next;
1005 				}
1006 				pmap_pvlist_check(mdpg);
1007 			}
1008 		}
1009 		pmap_pvlist_check(mdpg);
1010 		VM_PAGEMD_PVLIST_UNLOCK(mdpg);
1011 		kpreempt_enable();
1012 		break;
1013 
1014 	/* remove_all */
1015 	default:
1016 		pmap_page_remove(pg);
1017 	}
1018 
1019 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1020 }
1021 
1022 static bool
1023 pmap_pte_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep,
1024 	uintptr_t flags)
1025 {
1026 	const vm_prot_t prot = (flags & VM_PROT_ALL);
1027 
1028 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1029 	UVMHIST_LOG(pmaphist, "(pmap=%p %sva=%#"PRIxVADDR"..%#"PRIxVADDR,
1030 	    pmap, (pmap == pmap_kernel() ? "(kernel) " : ""), sva, eva);
1031 	UVMHIST_LOG(pmaphist, "ptep=%p, flags(npte)=%#"PRIxPTR")",
1032 	    ptep, flags, 0, 0);
1033 
1034 	KASSERT(kpreempt_disabled());
1035 	/*
1036 	 * Change protection on every valid mapping within this segment.
1037 	 */
1038 	for (; sva < eva; sva += NBPG, ptep++) {
1039 		pt_entry_t pte = *ptep;
1040 		if (!pte_valid_p(pte))
1041 			continue;
1042 		struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte));
1043 		if (pg != NULL && pte_modified_p(pte)) {
1044 			struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
1045 			if (VM_PAGEMD_EXECPAGE_P(mdpg)) {
1046 				KASSERT(mdpg->mdpg_first.pv_pmap != NULL);
1047 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1048 				if (VM_PAGEMD_CACHED_P(mdpg)) {
1049 #endif
1050 					UVMHIST_LOG(pmapexechist,
1051 					    "pg %p (pa %#"PRIxPADDR"): %s",
1052 					    pg, VM_PAGE_TO_PHYS(pg),
1053 					    "syncicached performed", 0);
1054 					pmap_page_syncicache(pg);
1055 					PMAP_COUNT(exec_synced_protect);
1056 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1057 				}
1058 #endif
1059 			}
1060 		}
1061 		pte = pte_prot_downgrade(pte, prot);
1062 		if (*ptep != pte) {
1063 			pmap_md_tlb_miss_lock_enter();
1064 			*ptep = pte;
1065 			/*
1066 			 * Update the TLB if needed.
1067 			 */
1068 			pmap_tlb_update_addr(pmap, sva, pte, PMAP_TLB_NEED_IPI);
1069 			pmap_md_tlb_miss_lock_exit();
1070 		}
1071 	}
1072 
1073 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1074 
1075 	return false;
1076 }
1077 
1078 /*
1079  *	Set the physical protection on the
1080  *	specified range of this map as requested.
1081  */
1082 void
1083 pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
1084 {
1085 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1086 	UVMHIST_LOG(pmaphist,
1087 	    "(pmap=%p, va=%#"PRIxVADDR"..%#"PRIxVADDR", prot=%u)",
1088 	    pmap, sva, eva, prot);
1089 	PMAP_COUNT(protect);
1090 
1091 	if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
1092 		pmap_remove(pmap, sva, eva);
1093 		UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1094 		return;
1095 	}
1096 
1097 	/*
1098 	 * Change protection on every valid mapping within this segment.
1099 	 */
1100 	kpreempt_disable();
1101 	pmap_addr_range_check(pmap, sva, eva, __func__);
1102 	pmap_pte_process(pmap, sva, eva, pmap_pte_protect, prot);
1103 	kpreempt_enable();
1104 
1105 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1106 }
1107 
1108 #if defined(PMAP_VIRTUAL_CACHE_ALIASES) && !defined(PMAP_NO_PV_UNCACHED)
1109 /*
1110  *	pmap_page_cache:
1111  *
1112  *	Change all mappings of a managed page to cached/uncached.
1113  */
1114 void
1115 pmap_page_cache(struct vm_page *pg, bool cached)
1116 {
1117 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
1118 
1119 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1120 	UVMHIST_LOG(pmaphist, "(pg=%p (pa %#"PRIxPADDR") cached=%s)",
1121 	    pg, VM_PAGE_TO_PHYS(pg), cached ? "true" : "false", 0);
1122 
1123 	KASSERT(kpreempt_disabled());
1124 	KASSERT(VM_PAGEMD_PVLIST_LOCKED_P(mdpg));
1125 
1126 	if (cached) {
1127 		pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED);
1128 		PMAP_COUNT(page_cache_restorations);
1129 	} else {
1130 		pmap_page_set_attributes(mdpg, VM_PAGEMD_UNCACHED);
1131 		PMAP_COUNT(page_cache_evictions);
1132 	}
1133 
1134 	for (pv_entry_t pv = &mdpg->mdpg_first; pv != NULL; pv = pv->pv_next) {
1135 		pmap_t pmap = pv->pv_pmap;
1136 		vaddr_t va = trunc_page(pv->pv_va);
1137 
1138 		KASSERT(pmap != NULL);
1139 		KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va));
1140 		pt_entry_t * const ptep = pmap_pte_lookup(pmap, va);
1141 		if (ptep == NULL)
1142 			continue;
1143 		pt_entry_t pte = *ptep;
1144 		if (pte_valid_p(pte)) {
1145 			pte = pte_cached_change(pte, cached);
1146 			pmap_md_tlb_miss_lock_enter();
1147 			*ptep = pte;
1148 			pmap_tlb_update_addr(pmap, va, pte, PMAP_TLB_NEED_IPI);
1149 			pmap_md_tlb_miss_lock_exit();
1150 		}
1151 	}
1152 
1153 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1154 }
1155 #endif	/* PMAP_VIRTUAL_CACHE_ALIASES && !PMAP_NO_PV_UNCACHED */
1156 
1157 /*
1158  *	Insert the given physical page (p) at
1159  *	the specified virtual address (v) in the
1160  *	target physical map with the protection requested.
1161  *
1162  *	If specified, the page will be wired down, meaning
1163  *	that the related pte can not be reclaimed.
1164  *
1165  *	NB:  This is the only routine which MAY NOT lazy-evaluate
1166  *	or lose information.  That is, this routine must actually
1167  *	insert this page into the given map NOW.
1168  */
1169 int
1170 pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
1171 {
1172 	const bool wired = (flags & PMAP_WIRED) != 0;
1173 	const bool is_kernel_pmap_p = (pmap == pmap_kernel());
1174 	u_int update_flags = (flags & VM_PROT_ALL) != 0 ? PMAP_TLB_INSERT : 0;
1175 #ifdef UVMHIST
1176 	struct kern_history * const histp =
1177 	    ((prot & VM_PROT_EXECUTE) ? &pmapexechist : &pmaphist);
1178 #endif
1179 
1180 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(*histp);
1181 #define VM_PROT_STRING(prot) \
1182 	&"\0     " \
1183 	 "(R)\0  " \
1184 	 "(W)\0  " \
1185 	 "(RW)\0 " \
1186 	 "(X)\0  " \
1187 	 "(RX)\0 " \
1188 	 "(WX)\0 " \
1189 	 "(RWX)\0"[UVM_PROTECTION(prot)*6]
1190 	UVMHIST_LOG(*histp, "(pmap=%p, va=%#"PRIxVADDR", pa=%#"PRIxPADDR,
1191 	    pmap, va, pa, 0);
1192 	UVMHIST_LOG(*histp, "prot=%#x%s flags=%#x%s)",
1193 	    prot, VM_PROT_STRING(prot), flags, VM_PROT_STRING(flags));
1194 
1195 	const bool good_color = PMAP_PAGE_COLOROK_P(pa, va);
1196 	if (is_kernel_pmap_p) {
1197 		PMAP_COUNT(kernel_mappings);
1198 		if (!good_color)
1199 			PMAP_COUNT(kernel_mappings_bad);
1200 	} else {
1201 		PMAP_COUNT(user_mappings);
1202 		if (!good_color)
1203 			PMAP_COUNT(user_mappings_bad);
1204 	}
1205 	pmap_addr_range_check(pmap, va, va, __func__);
1206 
1207 	KASSERTMSG(prot & VM_PROT_READ, "no READ (%#x) in prot %#x",
1208 	    VM_PROT_READ, prot);
1209 
1210 	struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
1211 	struct vm_page_md * const mdpg = (pg ? VM_PAGE_TO_MD(pg) : NULL);
1212 
1213 	if (pg) {
1214 		/* Set page referenced/modified status based on flags */
1215 		if (flags & VM_PROT_WRITE) {
1216 			pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED|VM_PAGEMD_REFERENCED);
1217 		} else if (flags & VM_PROT_ALL) {
1218 			pmap_page_set_attributes(mdpg, VM_PAGEMD_REFERENCED);
1219 		}
1220 
1221 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1222 		if (!VM_PAGEMD_CACHED_P(mdpg)) {
1223 			flags |= PMAP_NOCACHE;
1224 			PMAP_COUNT(uncached_mappings);
1225 		}
1226 #endif
1227 
1228 		PMAP_COUNT(managed_mappings);
1229 	} else {
1230 		/*
1231 		 * Assumption: if it is not part of our managed memory
1232 		 * then it must be device memory which may be volatile.
1233 		 */
1234 		if ((flags & PMAP_CACHE_MASK) == 0)
1235 			flags |= PMAP_NOCACHE;
1236 		PMAP_COUNT(unmanaged_mappings);
1237 	}
1238 
1239 	pt_entry_t npte = pte_make_enter(pa, mdpg, prot, flags,
1240 	    is_kernel_pmap_p);
1241 
1242 	kpreempt_disable();
1243 
1244 	pt_entry_t * const ptep = pmap_pte_reserve(pmap, va, flags);
1245 	if (__predict_false(ptep == NULL)) {
1246 		kpreempt_enable();
1247 		UVMHIST_LOG(*histp, " <-- ENOMEM", 0, 0, 0, 0);
1248 		return ENOMEM;
1249 	}
1250 	const pt_entry_t opte = *ptep;
1251 
1252 	/* Done after case that may sleep/return. */
1253 	if (pg)
1254 		pmap_enter_pv(pmap, va, pg, &npte, 0);
1255 
1256 	/*
1257 	 * Now validate mapping with desired protection/wiring.
1258 	 * Assume uniform modified and referenced status for all
1259 	 * MIPS pages in a MACH page.
1260 	 */
1261 	if (wired) {
1262 		pmap->pm_stats.wired_count++;
1263 		npte = pte_wire_entry(npte);
1264 	}
1265 
1266 	UVMHIST_LOG(*histp, "new pte %#"PRIxPTE" (pa %#"PRIxPADDR")",
1267 	    pte_value(npte), pa, 0, 0);
1268 
1269 	if (pte_valid_p(opte) && pte_to_paddr(opte) != pa) {
1270 		pmap_remove(pmap, va, va + NBPG);
1271 		PMAP_COUNT(user_mappings_changed);
1272 	}
1273 
1274 	KASSERT(pte_valid_p(npte));
1275 	const bool resident = pte_valid_p(opte);
1276 	if (resident) {
1277 		update_flags |= PMAP_TLB_NEED_IPI;
1278 	} else {
1279 		pmap->pm_stats.resident_count++;
1280 	}
1281 
1282 	pmap_md_tlb_miss_lock_enter();
1283 	*ptep = npte;
1284 	pmap_tlb_update_addr(pmap, va, npte, update_flags);
1285 	pmap_md_tlb_miss_lock_exit();
1286 	kpreempt_enable();
1287 
1288 	if (pg != NULL && (prot == (VM_PROT_READ | VM_PROT_EXECUTE))) {
1289 		KASSERT(mdpg != NULL);
1290 		PMAP_COUNT(exec_mappings);
1291 		if (!VM_PAGEMD_EXECPAGE_P(mdpg) && pte_cached_p(npte)) {
1292 			if (!pte_deferred_exec_p(npte)) {
1293 				UVMHIST_LOG(*histp,
1294 				    "va=%#"PRIxVADDR" pg %p: %s syncicache%s",
1295 				    va, pg, "immediate", "");
1296 				pmap_page_syncicache(pg);
1297 				pmap_page_set_attributes(mdpg,
1298 				    VM_PAGEMD_EXECPAGE);
1299 				PMAP_COUNT(exec_synced_mappings);
1300 			} else {
1301 				UVMHIST_LOG(*histp, "va=%#"PRIxVADDR
1302 				    " pg %p: %s syncicache: pte %#x",
1303 				    va, pg, "defer", npte);
1304 			}
1305 		} else {
1306 			UVMHIST_LOG(*histp,
1307 			    "va=%#"PRIxVADDR" pg %p: %s syncicache%s",
1308 			    va, pg, "no",
1309 			    (pte_cached_p(npte)
1310 				? " (already exec)"
1311 				: " (uncached)"));
1312 		}
1313 	} else if (pg != NULL && (prot & VM_PROT_EXECUTE)) {
1314 		KASSERT(mdpg != NULL);
1315 		KASSERT(prot & VM_PROT_WRITE);
1316 		PMAP_COUNT(exec_mappings);
1317 		pmap_page_syncicache(pg);
1318 		pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE);
1319 		UVMHIST_LOG(*histp,
1320 		    "va=%#"PRIxVADDR" pg %p: %s syncicache%s",
1321 		    va, pg, "immediate", " (writeable)");
1322 	}
1323 
1324 	UVMHIST_LOG(*histp, " <-- 0 (OK)", 0, 0, 0, 0);
1325 	return 0;
1326 }
1327 
1328 void
1329 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
1330 {
1331 	pmap_t pmap = pmap_kernel();
1332 	struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
1333 	struct vm_page_md * const mdpg = (pg ? VM_PAGE_TO_MD(pg) : NULL);
1334 
1335 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1336 	UVMHIST_LOG(pmaphist,
1337 	    "(va=%#"PRIxVADDR", pa=%#"PRIxPADDR", prot=%u, flags=%#x)",
1338 	    va, pa, prot, flags);
1339 	PMAP_COUNT(kenter_pa);
1340 
1341 	if (mdpg == NULL) {
1342 		PMAP_COUNT(kenter_pa_unmanaged);
1343 		if ((flags & PMAP_CACHE_MASK) == 0)
1344 			flags |= PMAP_NOCACHE;
1345 	} else {
1346 		if ((flags & PMAP_NOCACHE) == 0 && !PMAP_PAGE_COLOROK_P(pa, va))
1347 			PMAP_COUNT(kenter_pa_bad);
1348 	}
1349 
1350 	pt_entry_t npte = pte_make_kenter_pa(pa, mdpg, prot, flags);
1351 	kpreempt_disable();
1352 	pt_entry_t * const ptep = pmap_pte_lookup(pmap, va);
1353 	KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va,
1354 	    pmap_limits.virtual_end);
1355 	KASSERT(!pte_valid_p(*ptep));
1356 
1357 	/*
1358 	 * No need to track non-managed pages or PMAP_KMPAGEs pages for aliases
1359 	 */
1360 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1361 	if (pg != NULL && (flags & PMAP_KMPAGE) == 0
1362 	    && pmap_md_virtual_cache_aliasing_p()) {
1363 		pmap_enter_pv(pmap, va, pg, &npte, PV_KENTER);
1364 	}
1365 #endif
1366 
1367 	/*
1368 	 * We have the option to force this mapping into the TLB but we
1369 	 * don't.  Instead let the next reference to the page do it.
1370 	 */
1371 	pmap_md_tlb_miss_lock_enter();
1372 	*ptep = npte;
1373 	pmap_tlb_update_addr(pmap_kernel(), va, npte, 0);
1374 	pmap_md_tlb_miss_lock_exit();
1375 	kpreempt_enable();
1376 #if DEBUG > 1
1377 	for (u_int i = 0; i < PAGE_SIZE / sizeof(long); i++) {
1378 		if (((long *)va)[i] != ((long *)pa)[i])
1379 			panic("%s: contents (%lx) of va %#"PRIxVADDR
1380 			    " != contents (%lx) of pa %#"PRIxPADDR, __func__,
1381 			    ((long *)va)[i], va, ((long *)pa)[i], pa);
1382 	}
1383 #endif
1384 
1385 	UVMHIST_LOG(pmaphist, " <-- done (ptep=%p)", ptep, 0, 0, 0);
1386 }
1387 
1388 /*
1389  *	Remove the given range of addresses from the kernel map.
1390  *
1391  *	It is assumed that the start and end are properly
1392  *	rounded to the page size.
1393  */
1394 
1395 static bool
1396 pmap_pte_kremove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep,
1397 	uintptr_t flags)
1398 {
1399 	const pt_entry_t new_pte = pte_nv_entry(true);
1400 
1401 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1402 	UVMHIST_LOG(pmaphist,
1403 	    "(pmap=%p, sva=%#"PRIxVADDR", eva=%#"PRIxVADDR", ptep=%p)",
1404 	    pmap, sva, eva, ptep);
1405 
1406 	KASSERT(kpreempt_disabled());
1407 
1408 	for (; sva < eva; sva += NBPG, ptep++) {
1409 		pt_entry_t pte = *ptep;
1410 		if (!pte_valid_p(pte))
1411 			continue;
1412 
1413 		PMAP_COUNT(kremove_pages);
1414 		struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte));
1415 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1416 		if (pg != NULL && pmap_md_virtual_cache_aliasing_p()) {
1417 			pmap_remove_pv(pmap, sva, pg, !pte_readonly_p(pte));
1418 		}
1419 #endif
1420 
1421 		pmap_md_tlb_miss_lock_enter();
1422 		*ptep = new_pte;
1423 		pmap_tlb_invalidate_addr(pmap, sva);
1424 		pmap_md_tlb_miss_lock_exit();
1425 	}
1426 
1427 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1428 
1429 	return false;
1430 }
1431 
1432 void
1433 pmap_kremove(vaddr_t va, vsize_t len)
1434 {
1435 	const vaddr_t sva = trunc_page(va);
1436 	const vaddr_t eva = round_page(va + len);
1437 
1438 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1439 	UVMHIST_LOG(pmaphist, "(va=%#"PRIxVADDR", len=%#"PRIxVSIZE")",
1440 	    va, len, 0, 0);
1441 
1442 	kpreempt_disable();
1443 	pmap_pte_process(pmap_kernel(), sva, eva, pmap_pte_kremove, 0);
1444 	kpreempt_enable();
1445 
1446 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1447 }
1448 
1449 void
1450 pmap_remove_all(struct pmap *pmap)
1451 {
1452 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1453 	UVMHIST_LOG(pmaphist, "(pm=%p)", pmap, 0, 0, 0);
1454 
1455 	KASSERT(pmap != pmap_kernel());
1456 
1457 	kpreempt_disable();
1458 	/*
1459 	 * Free all of our ASIDs which means we can skip doing all the
1460 	 * tlb_invalidate_addrs().
1461 	 */
1462 	pmap_md_tlb_miss_lock_enter();
1463 #ifdef MULTIPROCESSOR
1464 	// This should be the last CPU with this pmap onproc
1465 	KASSERT(!kcpuset_isotherset(pmap->pm_onproc, cpu_index(curcpu())));
1466 	if (kcpuset_isset(pmap->pm_onproc, cpu_index(curcpu())))
1467 #endif
1468 		pmap_tlb_asid_deactivate(pmap);
1469 #ifdef MULTIPROCESSOR
1470 	KASSERT(kcpuset_iszero(pmap->pm_onproc));
1471 #endif
1472 	pmap_tlb_asid_release_all(pmap);
1473 	pmap_md_tlb_miss_lock_exit();
1474 	pmap->pm_flags |= PMAP_DEFERRED_ACTIVATE;
1475 
1476 #ifdef PMAP_FAULTINFO
1477 	curpcb->pcb_faultinfo.pfi_faultaddr = 0;
1478 	curpcb->pcb_faultinfo.pfi_repeats = 0;
1479 	curpcb->pcb_faultinfo.pfi_faultpte = NULL;
1480 #endif
1481 	kpreempt_enable();
1482 
1483 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1484 }
1485 
1486 /*
1487  *	Routine:	pmap_unwire
1488  *	Function:	Clear the wired attribute for a map/virtual-address
1489  *			pair.
1490  *	In/out conditions:
1491  *			The mapping must already exist in the pmap.
1492  */
1493 void
1494 pmap_unwire(pmap_t pmap, vaddr_t va)
1495 {
1496 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1497 	UVMHIST_LOG(pmaphist, "(pmap=%p, va=%#"PRIxVADDR")", pmap, va, 0, 0);
1498 	PMAP_COUNT(unwire);
1499 
1500 	/*
1501 	 * Don't need to flush the TLB since PG_WIRED is only in software.
1502 	 */
1503 	kpreempt_disable();
1504 	pmap_addr_range_check(pmap, va, va, __func__);
1505 	pt_entry_t * const ptep = pmap_pte_lookup(pmap, va);
1506 	KASSERTMSG(ptep != NULL, "pmap %p va %#"PRIxVADDR" invalid STE",
1507 	    pmap, va);
1508 	pt_entry_t pte = *ptep;
1509 	KASSERTMSG(pte_valid_p(pte),
1510 	    "pmap %p va %#"PRIxVADDR" invalid PTE %#"PRIxPTE" @ %p",
1511 	    pmap, va, pte_value(pte), ptep);
1512 
1513 	if (pte_wired_p(pte)) {
1514 		pmap_md_tlb_miss_lock_enter();
1515 		*ptep = pte_unwire_entry(pte);
1516 		pmap_md_tlb_miss_lock_exit();
1517 		pmap->pm_stats.wired_count--;
1518 	}
1519 #ifdef DIAGNOSTIC
1520 	else {
1521 		printf("%s: wiring for pmap %p va %#"PRIxVADDR" unchanged!\n",
1522 		    __func__, pmap, va);
1523 	}
1524 #endif
1525 	kpreempt_enable();
1526 
1527 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1528 }
1529 
1530 /*
1531  *	Routine:	pmap_extract
1532  *	Function:
1533  *		Extract the physical page address associated
1534  *		with the given map/virtual_address pair.
1535  */
1536 bool
1537 pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap)
1538 {
1539 	paddr_t pa;
1540 
1541 	if (pmap == pmap_kernel()) {
1542 		if (pmap_md_direct_mapped_vaddr_p(va)) {
1543 			pa = pmap_md_direct_mapped_vaddr_to_paddr(va);
1544 			goto done;
1545 		}
1546 		if (pmap_md_io_vaddr_p(va))
1547 			panic("pmap_extract: io address %#"PRIxVADDR"", va);
1548 
1549 		if (va >= pmap_limits.virtual_end)
1550 			panic("%s: illegal kernel mapped address %#"PRIxVADDR,
1551 			    __func__, va);
1552 	}
1553 	kpreempt_disable();
1554 	const pt_entry_t * const ptep = pmap_pte_lookup(pmap, va);
1555 	if (ptep == NULL || !pte_valid_p(*ptep)) {
1556 		kpreempt_enable();
1557 		return false;
1558 	}
1559 	pa = pte_to_paddr(*ptep) | (va & PGOFSET);
1560 	kpreempt_enable();
1561 done:
1562 	if (pap != NULL) {
1563 		*pap = pa;
1564 	}
1565 	return true;
1566 }
1567 
1568 /*
1569  *	Copy the range specified by src_addr/len
1570  *	from the source map to the range dst_addr/len
1571  *	in the destination map.
1572  *
1573  *	This routine is only advisory and need not do anything.
1574  */
1575 void
1576 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr, vsize_t len,
1577     vaddr_t src_addr)
1578 {
1579 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1580 	PMAP_COUNT(copy);
1581 }
1582 
1583 /*
1584  *	pmap_clear_reference:
1585  *
1586  *	Clear the reference bit on the specified physical page.
1587  */
1588 bool
1589 pmap_clear_reference(struct vm_page *pg)
1590 {
1591 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
1592 
1593 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1594 	UVMHIST_LOG(pmaphist, "(pg=%p (pa %#"PRIxPADDR"))",
1595 	   pg, VM_PAGE_TO_PHYS(pg), 0,0);
1596 
1597 	bool rv = pmap_page_clear_attributes(mdpg, VM_PAGEMD_REFERENCED);
1598 
1599 	UVMHIST_LOG(pmaphist, " <-- %s", rv ? "true" : "false", 0, 0, 0);
1600 
1601 	return rv;
1602 }
1603 
1604 /*
1605  *	pmap_is_referenced:
1606  *
1607  *	Return whether or not the specified physical page is referenced
1608  *	by any physical maps.
1609  */
1610 bool
1611 pmap_is_referenced(struct vm_page *pg)
1612 {
1613 	return VM_PAGEMD_REFERENCED_P(VM_PAGE_TO_MD(pg));
1614 }
1615 
1616 /*
1617  *	Clear the modify bits on the specified physical page.
1618  */
1619 bool
1620 pmap_clear_modify(struct vm_page *pg)
1621 {
1622 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
1623 	pv_entry_t pv = &mdpg->mdpg_first;
1624 	pv_entry_t pv_next;
1625 
1626 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1627 	UVMHIST_LOG(pmaphist, "(pg=%p (%#"PRIxPADDR"))",
1628 	    pg, VM_PAGE_TO_PHYS(pg), 0,0);
1629 	PMAP_COUNT(clear_modify);
1630 
1631 	if (VM_PAGEMD_EXECPAGE_P(mdpg)) {
1632 		if (pv->pv_pmap == NULL) {
1633 			UVMHIST_LOG(pmapexechist,
1634 			    "pg %p (pa %#"PRIxPADDR"): %s",
1635 			    pg, VM_PAGE_TO_PHYS(pg), "execpage cleared", 0);
1636 			pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE);
1637 			PMAP_COUNT(exec_uncached_clear_modify);
1638 		} else {
1639 			UVMHIST_LOG(pmapexechist,
1640 			    "pg %p (pa %#"PRIxPADDR"): %s",
1641 			    pg, VM_PAGE_TO_PHYS(pg), "syncicache performed", 0);
1642 			pmap_page_syncicache(pg);
1643 			PMAP_COUNT(exec_synced_clear_modify);
1644 		}
1645 	}
1646 	if (!pmap_page_clear_attributes(mdpg, VM_PAGEMD_MODIFIED)) {
1647 		UVMHIST_LOG(pmaphist, " <-- false", 0, 0, 0, 0);
1648 		return false;
1649 	}
1650 	if (pv->pv_pmap == NULL) {
1651 		UVMHIST_LOG(pmaphist, " <-- true (no mappings)", 0, 0, 0, 0);
1652 		return true;
1653 	}
1654 
1655 	/*
1656 	 * remove write access from any pages that are dirty
1657 	 * so we can tell if they are written to again later.
1658 	 * flush the VAC first if there is one.
1659 	 */
1660 	kpreempt_disable();
1661 	KASSERT(!VM_PAGEMD_PVLIST_LOCKED_P(mdpg));
1662 	VM_PAGEMD_PVLIST_READLOCK(mdpg);
1663 	pmap_pvlist_check(mdpg);
1664 	for (; pv != NULL; pv = pv_next) {
1665 		pmap_t pmap = pv->pv_pmap;
1666 		vaddr_t va = trunc_page(pv->pv_va);
1667 
1668 		pv_next = pv->pv_next;
1669 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1670 		if (pv->pv_va & PV_KENTER)
1671 			continue;
1672 #endif
1673 		pt_entry_t * const ptep = pmap_pte_lookup(pmap, va);
1674 		KASSERT(ptep);
1675 		pt_entry_t pte = pte_prot_nowrite(*ptep);
1676 		if (*ptep == pte) {
1677 			continue;
1678 		}
1679 		KASSERT(pte_valid_p(pte));
1680 		const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg);
1681 		pmap_md_tlb_miss_lock_enter();
1682 		*ptep = pte;
1683 		pmap_tlb_invalidate_addr(pmap, va);
1684 		pmap_md_tlb_miss_lock_exit();
1685 		pmap_update(pmap);
1686 		if (__predict_false(gen != VM_PAGEMD_PVLIST_READLOCK(mdpg))) {
1687 			/*
1688 			 * The list changed!  So restart from the beginning.
1689 			 */
1690 			pv_next = &mdpg->mdpg_first;
1691 			pmap_pvlist_check(mdpg);
1692 		}
1693 	}
1694 	pmap_pvlist_check(mdpg);
1695 	VM_PAGEMD_PVLIST_UNLOCK(mdpg);
1696 	kpreempt_enable();
1697 
1698 	UVMHIST_LOG(pmaphist, " <-- true (mappings changed)", 0, 0, 0, 0);
1699 	return true;
1700 }
1701 
1702 /*
1703  *	pmap_is_modified:
1704  *
1705  *	Return whether or not the specified physical page is modified
1706  *	by any physical maps.
1707  */
1708 bool
1709 pmap_is_modified(struct vm_page *pg)
1710 {
1711 	return VM_PAGEMD_MODIFIED_P(VM_PAGE_TO_MD(pg));
1712 }
1713 
1714 /*
1715  *	pmap_set_modified:
1716  *
1717  *	Sets the page modified reference bit for the specified page.
1718  */
1719 void
1720 pmap_set_modified(paddr_t pa)
1721 {
1722 	struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
1723 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
1724 	pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED|VM_PAGEMD_REFERENCED);
1725 }
1726 
1727 /******************** pv_entry management ********************/
1728 
1729 static void
1730 pmap_pvlist_check(struct vm_page_md *mdpg)
1731 {
1732 #ifdef DEBUG
1733 	pv_entry_t pv = &mdpg->mdpg_first;
1734 	if (pv->pv_pmap != NULL) {
1735 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1736 		const u_int colormask = uvmexp.colormask;
1737 		u_int colors = 0;
1738 #endif
1739 		for (; pv != NULL; pv = pv->pv_next) {
1740 			KASSERT(pv->pv_pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(pv->pv_va));
1741 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1742 			colors |= __BIT(atop(pv->pv_va) & colormask);
1743 #endif
1744 		}
1745 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1746 		// Assert there if there more than 1 color mapped, that they
1747 		// are uncached.
1748 		KASSERTMSG(!pmap_md_virtual_cache_aliasing_p()
1749 		    || colors == 0 || (colors & (colors-1)) == 0
1750 		    || VM_PAGEMD_UNCACHED_P(mdpg), "colors=%#x uncached=%u",
1751 		    colors, VM_PAGEMD_UNCACHED_P(mdpg));
1752 #endif
1753 	}
1754 #endif /* DEBUG */
1755 }
1756 
1757 /*
1758  * Enter the pmap and virtual address into the
1759  * physical to virtual map table.
1760  */
1761 void
1762 pmap_enter_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, pt_entry_t *nptep,
1763     u_int flags)
1764 {
1765 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
1766 	pv_entry_t pv, npv, apv;
1767 #ifdef UVMHIST
1768 	bool first = false;
1769 #endif
1770 
1771 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1772 	UVMHIST_LOG(pmaphist,
1773 	    "(pmap=%p va=%#"PRIxVADDR" pg=%p (%#"PRIxPADDR")",
1774 	    pmap, va, pg, VM_PAGE_TO_PHYS(pg));
1775 	UVMHIST_LOG(pmaphist, "nptep=%p (%#"PRIxPTE"))",
1776 	    nptep, pte_value(*nptep), 0, 0);
1777 
1778 	KASSERT(kpreempt_disabled());
1779 	KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va));
1780 	KASSERTMSG(pmap != pmap_kernel() || !pmap_md_io_vaddr_p(va),
1781 	    "va %#"PRIxVADDR, va);
1782 
1783 	apv = NULL;
1784 	VM_PAGEMD_PVLIST_LOCK(mdpg);
1785 again:
1786 	pv = &mdpg->mdpg_first;
1787 	pmap_pvlist_check(mdpg);
1788 	if (pv->pv_pmap == NULL) {
1789 		KASSERT(pv->pv_next == NULL);
1790 		/*
1791 		 * No entries yet, use header as the first entry
1792 		 */
1793 		PMAP_COUNT(primary_mappings);
1794 		PMAP_COUNT(mappings);
1795 #ifdef UVMHIST
1796 		first = true;
1797 #endif
1798 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1799 		KASSERT(VM_PAGEMD_CACHED_P(mdpg));
1800 		// If the new mapping has an incompatible color the last
1801 		// mapping of this page, clean the page before using it.
1802 		if (!PMAP_PAGE_COLOROK_P(va, pv->pv_va)) {
1803 			pmap_md_vca_clean(pg, PMAP_WBINV);
1804 		}
1805 #endif
1806 		pv->pv_pmap = pmap;
1807 		pv->pv_va = va | flags;
1808 	} else {
1809 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1810 		if (pmap_md_vca_add(pg, va, nptep)) {
1811 			goto again;
1812 		}
1813 #endif
1814 
1815 		/*
1816 		 * There is at least one other VA mapping this page.
1817 		 * Place this entry after the header.
1818 		 *
1819 		 * Note: the entry may already be in the table if
1820 		 * we are only changing the protection bits.
1821 		 */
1822 
1823 #ifdef PARANOIADIAG
1824 		const paddr_t pa = VM_PAGE_TO_PHYS(pg);
1825 #endif
1826 		for (npv = pv; npv; npv = npv->pv_next) {
1827 			if (pmap == npv->pv_pmap
1828 			    && va == trunc_page(npv->pv_va)) {
1829 #ifdef PARANOIADIAG
1830 				pt_entry_t *ptep = pmap_pte_lookup(pmap, va);
1831 				pt_entry_t pte = (ptep != NULL) ? *ptep : 0;
1832 				if (!pte_valid_p(pte) || pte_to_paddr(pte) != pa)
1833 					printf("%s: found va %#"PRIxVADDR
1834 					    " pa %#"PRIxPADDR
1835 					    " in pv_table but != %#"PRIxPTE"\n",
1836 					    __func__, va, pa, pte_value(pte));
1837 #endif
1838 				PMAP_COUNT(remappings);
1839 				VM_PAGEMD_PVLIST_UNLOCK(mdpg);
1840 				if (__predict_false(apv != NULL))
1841 					pmap_pv_free(apv);
1842 
1843 				UVMHIST_LOG(pmaphist, " <-- done pv=%p%s",
1844 				    pv, " (reused)", 0, 0);
1845 				return;
1846 			}
1847 		}
1848 		if (__predict_true(apv == NULL)) {
1849 			/*
1850 			 * To allocate a PV, we have to release the PVLIST lock
1851 			 * so get the page generation.  We allocate the PV, and
1852 			 * then reacquire the lock.
1853 			 */
1854 			pmap_pvlist_check(mdpg);
1855 			const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg);
1856 
1857 			apv = (pv_entry_t)pmap_pv_alloc();
1858 			if (apv == NULL)
1859 				panic("pmap_enter_pv: pmap_pv_alloc() failed");
1860 
1861 			/*
1862 			 * If the generation has changed, then someone else
1863 			 * tinkered with this page so we should start over.
1864 			 */
1865 			if (gen != VM_PAGEMD_PVLIST_LOCK(mdpg))
1866 				goto again;
1867 		}
1868 		npv = apv;
1869 		apv = NULL;
1870 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1871 		/*
1872 		 * If need to deal with virtual cache aliases, keep mappings
1873 		 * in the kernel pmap at the head of the list.  This allows
1874 		 * the VCA code to easily use them for cache operations if
1875 		 * present.
1876 		 */
1877 		pmap_t kpmap = pmap_kernel();
1878 		if (pmap != kpmap) {
1879 			while (pv->pv_pmap == kpmap && pv->pv_next != NULL) {
1880 				pv = pv->pv_next;
1881 			}
1882 		}
1883 #endif
1884 		npv->pv_va = va | flags;
1885 		npv->pv_pmap = pmap;
1886 		npv->pv_next = pv->pv_next;
1887 		pv->pv_next = npv;
1888 		PMAP_COUNT(mappings);
1889 	}
1890 	pmap_pvlist_check(mdpg);
1891 	VM_PAGEMD_PVLIST_UNLOCK(mdpg);
1892 	if (__predict_false(apv != NULL))
1893 		pmap_pv_free(apv);
1894 
1895 	UVMHIST_LOG(pmaphist, " <-- done pv=%p%s",
1896 	    pv, first ? " (first pv)" : "",0,0);
1897 }
1898 
1899 /*
1900  * Remove a physical to virtual address translation.
1901  * If cache was inhibited on this page, and there are no more cache
1902  * conflicts, restore caching.
1903  * Flush the cache if the last page is removed (should always be cached
1904  * at this point).
1905  */
1906 void
1907 pmap_remove_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, bool dirty)
1908 {
1909 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
1910 	pv_entry_t pv, npv;
1911 	bool last;
1912 
1913 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1914 	UVMHIST_LOG(pmaphist,
1915 	    "(pmap=%p, va=%#"PRIxVADDR", pg=%p (pa %#"PRIxPADDR")",
1916 	    pmap, va, pg, VM_PAGE_TO_PHYS(pg));
1917 	UVMHIST_LOG(pmaphist, "dirty=%s)", dirty ? "true" : "false", 0, 0, 0);
1918 
1919 	KASSERT(kpreempt_disabled());
1920 	KASSERT((va & PAGE_MASK) == 0);
1921 	pv = &mdpg->mdpg_first;
1922 
1923 	VM_PAGEMD_PVLIST_LOCK(mdpg);
1924 	pmap_pvlist_check(mdpg);
1925 
1926 	/*
1927 	 * If it is the first entry on the list, it is actually
1928 	 * in the header and we must copy the following entry up
1929 	 * to the header.  Otherwise we must search the list for
1930 	 * the entry.  In either case we free the now unused entry.
1931 	 */
1932 
1933 	last = false;
1934 	if (pmap == pv->pv_pmap && va == trunc_page(pv->pv_va)) {
1935 		npv = pv->pv_next;
1936 		if (npv) {
1937 			*pv = *npv;
1938 			KASSERT(pv->pv_pmap != NULL);
1939 		} else {
1940 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1941 			pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED);
1942 #endif
1943 			pv->pv_pmap = NULL;
1944 			last = true;	/* Last mapping removed */
1945 		}
1946 		PMAP_COUNT(remove_pvfirst);
1947 	} else {
1948 		for (npv = pv->pv_next; npv; pv = npv, npv = npv->pv_next) {
1949 			PMAP_COUNT(remove_pvsearch);
1950 			if (pmap == npv->pv_pmap && va == trunc_page(npv->pv_va))
1951 				break;
1952 		}
1953 		if (npv) {
1954 			pv->pv_next = npv->pv_next;
1955 		}
1956 	}
1957 
1958 	pmap_pvlist_check(mdpg);
1959 	VM_PAGEMD_PVLIST_UNLOCK(mdpg);
1960 
1961 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1962 	pmap_md_vca_remove(pg, va, dirty, last);
1963 #endif
1964 
1965 	/*
1966 	 * Free the pv_entry if needed.
1967 	 */
1968 	if (npv)
1969 		pmap_pv_free(npv);
1970 	if (VM_PAGEMD_EXECPAGE_P(mdpg) && dirty) {
1971 		if (last) {
1972 			/*
1973 			 * If this was the page's last mapping, we no longer
1974 			 * care about its execness.
1975 			 */
1976 			UVMHIST_LOG(pmapexechist,
1977 			    "pg %p (pa %#"PRIxPADDR")%s: %s",
1978 			    pg, VM_PAGE_TO_PHYS(pg),
1979 			    last ? " [last mapping]" : "",
1980 			    "execpage cleared");
1981 			pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE);
1982 			PMAP_COUNT(exec_uncached_remove);
1983 		} else {
1984 			/*
1985 			 * Someone still has it mapped as an executable page
1986 			 * so we must sync it.
1987 			 */
1988 			UVMHIST_LOG(pmapexechist,
1989 			    "pg %p (pa %#"PRIxPADDR")%s: %s",
1990 			    pg, VM_PAGE_TO_PHYS(pg),
1991 			    last ? " [last mapping]" : "",
1992 			    "performed syncicache");
1993 			pmap_page_syncicache(pg);
1994 			PMAP_COUNT(exec_synced_remove);
1995 		}
1996 	}
1997 
1998 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1999 }
2000 
2001 #if defined(MULTIPROCESSOR)
2002 struct pmap_pvlist_info {
2003 	kmutex_t *pli_locks[PAGE_SIZE / 32];
2004 	volatile u_int pli_lock_refs[PAGE_SIZE / 32];
2005 	volatile u_int pli_lock_index;
2006 	u_int pli_lock_mask;
2007 } pmap_pvlist_info;
2008 
2009 void
2010 pmap_pvlist_lock_init(size_t cache_line_size)
2011 {
2012 	struct pmap_pvlist_info * const pli = &pmap_pvlist_info;
2013 	const vaddr_t lock_page = uvm_pageboot_alloc(PAGE_SIZE);
2014 	vaddr_t lock_va = lock_page;
2015 	if (sizeof(kmutex_t) > cache_line_size) {
2016 		cache_line_size = roundup2(sizeof(kmutex_t), cache_line_size);
2017 	}
2018 	const size_t nlocks = PAGE_SIZE / cache_line_size;
2019 	KASSERT((nlocks & (nlocks - 1)) == 0);
2020 	/*
2021 	 * Now divide the page into a number of mutexes, one per cacheline.
2022 	 */
2023 	for (size_t i = 0; i < nlocks; lock_va += cache_line_size, i++) {
2024 		kmutex_t * const lock = (kmutex_t *)lock_va;
2025 		mutex_init(lock, MUTEX_DEFAULT, IPL_HIGH);
2026 		pli->pli_locks[i] = lock;
2027 	}
2028 	pli->pli_lock_mask = nlocks - 1;
2029 }
2030 
2031 kmutex_t *
2032 pmap_pvlist_lock_addr(struct vm_page_md *mdpg)
2033 {
2034 	struct pmap_pvlist_info * const pli = &pmap_pvlist_info;
2035 	kmutex_t *lock = mdpg->mdpg_lock;
2036 
2037 	/*
2038 	 * Allocate a lock on an as-needed basis.  This will hopefully give us
2039 	 * semi-random distribution not based on page color.
2040 	 */
2041 	if (__predict_false(lock == NULL)) {
2042 		size_t locknum = atomic_add_int_nv(&pli->pli_lock_index, 37);
2043 		size_t lockid = locknum & pli->pli_lock_mask;
2044 		kmutex_t * const new_lock = pli->pli_locks[lockid];
2045 		/*
2046 		 * Set the lock.  If some other thread already did, just use
2047 		 * the one they assigned.
2048 		 */
2049 		lock = atomic_cas_ptr(&mdpg->mdpg_lock, NULL, new_lock);
2050 		if (lock == NULL) {
2051 			lock = new_lock;
2052 			atomic_inc_uint(&pli->pli_lock_refs[lockid]);
2053 		}
2054 	}
2055 
2056 	/*
2057 	 * Now finally provide the lock.
2058 	 */
2059 	return lock;
2060 }
2061 #else /* !MULTIPROCESSOR */
2062 void
2063 pmap_pvlist_lock_init(size_t cache_line_size)
2064 {
2065 	mutex_init(&pmap_pvlist_mutex, MUTEX_DEFAULT, IPL_HIGH);
2066 }
2067 
2068 #ifdef MODULAR
2069 kmutex_t *
2070 pmap_pvlist_lock_addr(struct vm_page_md *mdpg)
2071 {
2072 	/*
2073 	 * We just use a global lock.
2074 	 */
2075 	if (__predict_false(mdpg->mdpg_lock == NULL)) {
2076 		mdpg->mdpg_lock = &pmap_pvlist_mutex;
2077 	}
2078 
2079 	/*
2080 	 * Now finally provide the lock.
2081 	 */
2082 	return mdpg->mdpg_lock;
2083 }
2084 #endif /* MODULAR */
2085 #endif /* !MULTIPROCESSOR */
2086 
2087 /*
2088  * pmap_pv_page_alloc:
2089  *
2090  *	Allocate a page for the pv_entry pool.
2091  */
2092 void *
2093 pmap_pv_page_alloc(struct pool *pp, int flags)
2094 {
2095 	struct vm_page * const pg = PMAP_ALLOC_POOLPAGE(UVM_PGA_USERESERVE);
2096 	if (pg == NULL)
2097 		return NULL;
2098 
2099 	return (void *)pmap_map_poolpage(VM_PAGE_TO_PHYS(pg));
2100 }
2101 
2102 /*
2103  * pmap_pv_page_free:
2104  *
2105  *	Free a pv_entry pool page.
2106  */
2107 void
2108 pmap_pv_page_free(struct pool *pp, void *v)
2109 {
2110 	vaddr_t va = (vaddr_t)v;
2111 
2112 	KASSERT(pmap_md_direct_mapped_vaddr_p(va));
2113 	const paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va);
2114 	struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
2115 	KASSERT(pg != NULL);
2116 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
2117 	kpreempt_disable();
2118 	pmap_md_vca_remove(pg, va, true, true);
2119 	kpreempt_enable();
2120 #endif
2121 	pmap_page_clear_attributes(VM_PAGE_TO_MD(pg), VM_PAGEMD_POOLPAGE);
2122 	uvm_pagefree(pg);
2123 }
2124 
2125 #ifdef PMAP_PREFER
2126 /*
2127  * Find first virtual address >= *vap that doesn't cause
2128  * a cache alias conflict.
2129  */
2130 void
2131 pmap_prefer(vaddr_t foff, vaddr_t *vap, vsize_t sz, int td)
2132 {
2133 	vsize_t prefer_mask = ptoa(uvmexp.colormask);
2134 
2135 	PMAP_COUNT(prefer_requests);
2136 
2137 	prefer_mask |= pmap_md_cache_prefer_mask();
2138 
2139 	if (prefer_mask) {
2140 		vaddr_t	va = *vap;
2141 		vsize_t d = (foff - va) & prefer_mask;
2142 		if (d) {
2143 			if (td)
2144 				*vap = trunc_page(va - ((-d) & prefer_mask));
2145 			else
2146 				*vap = round_page(va + d);
2147 			PMAP_COUNT(prefer_adjustments);
2148 		}
2149 	}
2150 }
2151 #endif /* PMAP_PREFER */
2152 
2153 #ifdef PMAP_MAP_POOLPAGE
2154 vaddr_t
2155 pmap_map_poolpage(paddr_t pa)
2156 {
2157 	struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
2158 	KASSERT(pg);
2159 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
2160 	pmap_page_set_attributes(mdpg, VM_PAGEMD_POOLPAGE);
2161 
2162 	return pmap_md_map_poolpage(pa, NBPG);
2163 }
2164 
2165 paddr_t
2166 pmap_unmap_poolpage(vaddr_t va)
2167 {
2168 	KASSERT(pmap_md_direct_mapped_vaddr_p(va));
2169 	paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va);
2170 
2171 	struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
2172 	KASSERT(pg != NULL);
2173 	pmap_page_clear_attributes(VM_PAGE_TO_MD(pg), VM_PAGEMD_POOLPAGE);
2174 	pmap_md_unmap_poolpage(va, NBPG);
2175 
2176 	return pa;
2177 }
2178 #endif /* PMAP_MAP_POOLPAGE */
2179