xref: /netbsd-src/sys/uvm/pmap/pmap.c (revision e89934bbf778a6d6d6894877c4da59d0c7835b0f)
1 /*	$NetBSD: pmap.c,v 1.27 2016/12/23 09:16:46 skrll Exp $	*/
2 
3 /*-
4  * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9  * NASA Ames Research Center and by Chris G. Demetriou.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30  * POSSIBILITY OF SUCH DAMAGE.
31  */
32 
33 /*
34  * Copyright (c) 1992, 1993
35  *	The Regents of the University of California.  All rights reserved.
36  *
37  * This code is derived from software contributed to Berkeley by
38  * the Systems Programming Group of the University of Utah Computer
39  * Science Department and Ralph Campbell.
40  *
41  * Redistribution and use in source and binary forms, with or without
42  * modification, are permitted provided that the following conditions
43  * are met:
44  * 1. Redistributions of source code must retain the above copyright
45  *    notice, this list of conditions and the following disclaimer.
46  * 2. Redistributions in binary form must reproduce the above copyright
47  *    notice, this list of conditions and the following disclaimer in the
48  *    documentation and/or other materials provided with the distribution.
49  * 3. Neither the name of the University nor the names of its contributors
50  *    may be used to endorse or promote products derived from this software
51  *    without specific prior written permission.
52  *
53  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
54  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
55  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
56  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
57  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
58  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
59  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
60  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
61  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
62  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
63  * SUCH DAMAGE.
64  *
65  *	@(#)pmap.c	8.4 (Berkeley) 1/26/94
66  */
67 
68 #include <sys/cdefs.h>
69 
70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.27 2016/12/23 09:16:46 skrll Exp $");
71 
72 /*
73  *	Manages physical address maps.
74  *
75  *	In addition to hardware address maps, this
76  *	module is called upon to provide software-use-only
77  *	maps which may or may not be stored in the same
78  *	form as hardware maps.  These pseudo-maps are
79  *	used to store intermediate results from copy
80  *	operations to and from address spaces.
81  *
82  *	Since the information managed by this module is
83  *	also stored by the logical address mapping module,
84  *	this module may throw away valid virtual-to-physical
85  *	mappings at almost any time.  However, invalidations
86  *	of virtual-to-physical mappings must be done as
87  *	requested.
88  *
89  *	In order to cope with hardware architectures which
90  *	make virtual-to-physical map invalidates expensive,
91  *	this module may delay invalidate or reduced protection
92  *	operations until such time as they are actually
93  *	necessary.  This module is given full information as
94  *	to which processors are currently using which maps,
95  *	and to when physical maps must be made correct.
96  */
97 
98 #include "opt_modular.h"
99 #include "opt_multiprocessor.h"
100 #include "opt_sysv.h"
101 
102 #define __PMAP_PRIVATE
103 
104 #include <sys/param.h>
105 #include <sys/atomic.h>
106 #include <sys/buf.h>
107 #include <sys/cpu.h>
108 #include <sys/mutex.h>
109 #include <sys/pool.h>
110 #include <sys/atomic.h>
111 #include <sys/mutex.h>
112 #include <sys/atomic.h>
113 
114 #include <uvm/uvm.h>
115 #include <uvm/uvm_physseg.h>
116 
117 #if defined(MULTIPROCESSOR) && defined(PMAP_VIRTUAL_CACHE_ALIASES) \
118     && !defined(PMAP_NO_PV_UNCACHED)
119 #error PMAP_VIRTUAL_CACHE_ALIASES with MULTIPROCESSOR requires \
120  PMAP_NO_PV_UNCACHED to be defined
121 #endif
122 
123 PMAP_COUNTER(remove_kernel_calls, "remove kernel calls");
124 PMAP_COUNTER(remove_kernel_pages, "kernel pages unmapped");
125 PMAP_COUNTER(remove_user_calls, "remove user calls");
126 PMAP_COUNTER(remove_user_pages, "user pages unmapped");
127 PMAP_COUNTER(remove_flushes, "remove cache flushes");
128 PMAP_COUNTER(remove_tlb_ops, "remove tlb ops");
129 PMAP_COUNTER(remove_pvfirst, "remove pv first");
130 PMAP_COUNTER(remove_pvsearch, "remove pv search");
131 
132 PMAP_COUNTER(prefer_requests, "prefer requests");
133 PMAP_COUNTER(prefer_adjustments, "prefer adjustments");
134 
135 PMAP_COUNTER(idlezeroed_pages, "pages idle zeroed");
136 
137 PMAP_COUNTER(kenter_pa, "kernel fast mapped pages");
138 PMAP_COUNTER(kenter_pa_bad, "kernel fast mapped pages (bad color)");
139 PMAP_COUNTER(kenter_pa_unmanaged, "kernel fast mapped unmanaged pages");
140 PMAP_COUNTER(kremove_pages, "kernel fast unmapped pages");
141 
142 PMAP_COUNTER(page_cache_evictions, "pages changed to uncacheable");
143 PMAP_COUNTER(page_cache_restorations, "pages changed to cacheable");
144 
145 PMAP_COUNTER(kernel_mappings_bad, "kernel pages mapped (bad color)");
146 PMAP_COUNTER(user_mappings_bad, "user pages mapped (bad color)");
147 PMAP_COUNTER(kernel_mappings, "kernel pages mapped");
148 PMAP_COUNTER(user_mappings, "user pages mapped");
149 PMAP_COUNTER(user_mappings_changed, "user mapping changed");
150 PMAP_COUNTER(kernel_mappings_changed, "kernel mapping changed");
151 PMAP_COUNTER(uncached_mappings, "uncached pages mapped");
152 PMAP_COUNTER(unmanaged_mappings, "unmanaged pages mapped");
153 PMAP_COUNTER(managed_mappings, "managed pages mapped");
154 PMAP_COUNTER(mappings, "pages mapped");
155 PMAP_COUNTER(remappings, "pages remapped");
156 PMAP_COUNTER(unmappings, "pages unmapped");
157 PMAP_COUNTER(primary_mappings, "page initial mappings");
158 PMAP_COUNTER(primary_unmappings, "page final unmappings");
159 PMAP_COUNTER(tlb_hit, "page mapping");
160 
161 PMAP_COUNTER(exec_mappings, "exec pages mapped");
162 PMAP_COUNTER(exec_synced_mappings, "exec pages synced");
163 PMAP_COUNTER(exec_synced_remove, "exec pages synced (PR)");
164 PMAP_COUNTER(exec_synced_clear_modify, "exec pages synced (CM)");
165 PMAP_COUNTER(exec_synced_page_protect, "exec pages synced (PP)");
166 PMAP_COUNTER(exec_synced_protect, "exec pages synced (P)");
167 PMAP_COUNTER(exec_uncached_page_protect, "exec pages uncached (PP)");
168 PMAP_COUNTER(exec_uncached_clear_modify, "exec pages uncached (CM)");
169 PMAP_COUNTER(exec_uncached_zero_page, "exec pages uncached (ZP)");
170 PMAP_COUNTER(exec_uncached_copy_page, "exec pages uncached (CP)");
171 PMAP_COUNTER(exec_uncached_remove, "exec pages uncached (PR)");
172 
173 PMAP_COUNTER(create, "creates");
174 PMAP_COUNTER(reference, "references");
175 PMAP_COUNTER(dereference, "dereferences");
176 PMAP_COUNTER(destroy, "destroyed");
177 PMAP_COUNTER(activate, "activations");
178 PMAP_COUNTER(deactivate, "deactivations");
179 PMAP_COUNTER(update, "updates");
180 #ifdef MULTIPROCESSOR
181 PMAP_COUNTER(shootdown_ipis, "shootdown IPIs");
182 #endif
183 PMAP_COUNTER(unwire, "unwires");
184 PMAP_COUNTER(copy, "copies");
185 PMAP_COUNTER(clear_modify, "clear_modifies");
186 PMAP_COUNTER(protect, "protects");
187 PMAP_COUNTER(page_protect, "page_protects");
188 
189 #define PMAP_ASID_RESERVED 0
190 CTASSERT(PMAP_ASID_RESERVED == 0);
191 
192 #ifndef PMAP_SEGTAB_ALIGN
193 #define PMAP_SEGTAB_ALIGN	/* nothing */
194 #endif
195 #ifdef _LP64
196 pmap_segtab_t	pmap_kstart_segtab PMAP_SEGTAB_ALIGN; /* first mid-level segtab for kernel */
197 #endif
198 pmap_segtab_t	pmap_kern_segtab PMAP_SEGTAB_ALIGN = { /* top level segtab for kernel */
199 #ifdef _LP64
200 	.seg_seg[(VM_MIN_KERNEL_ADDRESS & XSEGOFSET) >> SEGSHIFT] = &pmap_kstart_segtab,
201 #endif
202 };
203 
204 struct pmap_kernel kernel_pmap_store = {
205 	.kernel_pmap = {
206 		.pm_count = 1,
207 		.pm_segtab = &pmap_kern_segtab,
208 		.pm_minaddr = VM_MIN_KERNEL_ADDRESS,
209 		.pm_maxaddr = VM_MAX_KERNEL_ADDRESS,
210 	},
211 };
212 
213 struct pmap * const kernel_pmap_ptr = &kernel_pmap_store.kernel_pmap;
214 
215 struct pmap_limits pmap_limits = {	/* VA and PA limits */
216 	.virtual_start = VM_MIN_KERNEL_ADDRESS,
217 };
218 
219 #ifdef UVMHIST
220 static struct kern_history_ent pmapexechistbuf[10000];
221 static struct kern_history_ent pmaphistbuf[10000];
222 UVMHIST_DEFINE(pmapexechist);
223 UVMHIST_DEFINE(pmaphist);
224 #endif
225 
226 /*
227  * The pools from which pmap structures and sub-structures are allocated.
228  */
229 struct pool pmap_pmap_pool;
230 struct pool pmap_pv_pool;
231 
232 #ifndef PMAP_PV_LOWAT
233 #define	PMAP_PV_LOWAT	16
234 #endif
235 int	pmap_pv_lowat = PMAP_PV_LOWAT;
236 
237 bool	pmap_initialized = false;
238 #define	PMAP_PAGE_COLOROK_P(a, b) \
239 		((((int)(a) ^ (int)(b)) & pmap_page_colormask) == 0)
240 u_int	pmap_page_colormask;
241 
242 #define PAGE_IS_MANAGED(pa)	(pmap_initialized && uvm_pageismanaged(pa))
243 
244 #define PMAP_IS_ACTIVE(pm)						\
245 	((pm) == pmap_kernel() || 					\
246 	 (pm) == curlwp->l_proc->p_vmspace->vm_map.pmap)
247 
248 /* Forward function declarations */
249 void pmap_page_remove(struct vm_page *);
250 static void pmap_pvlist_check(struct vm_page_md *);
251 void pmap_remove_pv(pmap_t, vaddr_t, struct vm_page *, bool);
252 void pmap_enter_pv(pmap_t, vaddr_t, struct vm_page *, pt_entry_t *, u_int);
253 
254 /*
255  * PV table management functions.
256  */
257 void	*pmap_pv_page_alloc(struct pool *, int);
258 void	pmap_pv_page_free(struct pool *, void *);
259 
260 struct pool_allocator pmap_pv_page_allocator = {
261 	pmap_pv_page_alloc, pmap_pv_page_free, 0,
262 };
263 
264 #define	pmap_pv_alloc()		pool_get(&pmap_pv_pool, PR_NOWAIT)
265 #define	pmap_pv_free(pv)	pool_put(&pmap_pv_pool, (pv))
266 
267 #if !defined(MULTIPROCESSOR) || !defined(PMAP_MD_NEED_TLB_MISS_LOCK)
268 #define	pmap_md_tlb_miss_lock_enter()	do { } while(/*CONSTCOND*/0)
269 #define	pmap_md_tlb_miss_lock_exit()	do { } while(/*CONSTCOND*/0)
270 #endif /* !MULTIPROCESSOR || !PMAP_MD_NEED_TLB_MISS_LOCK */
271 
272 #ifndef MULTIPROCESSOR
273 kmutex_t pmap_pvlist_mutex	__cacheline_aligned;
274 #endif
275 
276 /*
277  * Debug functions.
278  */
279 
280 #ifdef DEBUG
281 static inline void
282 pmap_asid_check(pmap_t pm, const char *func)
283 {
284 	if (!PMAP_IS_ACTIVE(pm))
285 		return;
286 
287 	struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(curcpu()));
288 	tlb_asid_t asid = tlb_get_asid();
289 	if (asid != pai->pai_asid)
290 		panic("%s: inconsistency for active TLB update: %u <-> %u",
291 		    func, asid, pai->pai_asid);
292 }
293 #endif
294 
295 static void
296 pmap_addr_range_check(pmap_t pmap, vaddr_t sva, vaddr_t eva, const char *func)
297 {
298 #ifdef DEBUG
299 	if (pmap == pmap_kernel()) {
300 		if (sva < VM_MIN_KERNEL_ADDRESS)
301 			panic("%s: kva %#"PRIxVADDR" not in range",
302 			    func, sva);
303 		if (eva >= pmap_limits.virtual_end)
304 			panic("%s: kva %#"PRIxVADDR" not in range",
305 			    func, eva);
306 	} else {
307 		if (eva > VM_MAXUSER_ADDRESS)
308 			panic("%s: uva %#"PRIxVADDR" not in range",
309 			    func, eva);
310 		pmap_asid_check(pmap, func);
311 	}
312 #endif
313 }
314 
315 /*
316  * Misc. functions.
317  */
318 
319 bool
320 pmap_page_clear_attributes(struct vm_page_md *mdpg, u_int clear_attributes)
321 {
322 	volatile unsigned long * const attrp = &mdpg->mdpg_attrs;
323 #ifdef MULTIPROCESSOR
324 	for (;;) {
325 		u_int old_attr = *attrp;
326 		if ((old_attr & clear_attributes) == 0)
327 			return false;
328 		u_int new_attr = old_attr & ~clear_attributes;
329 		if (old_attr == atomic_cas_ulong(attrp, old_attr, new_attr))
330 			return true;
331 	}
332 #else
333 	unsigned long old_attr = *attrp;
334 	if ((old_attr & clear_attributes) == 0)
335 		return false;
336 	*attrp &= ~clear_attributes;
337 	return true;
338 #endif
339 }
340 
341 void
342 pmap_page_set_attributes(struct vm_page_md *mdpg, u_int set_attributes)
343 {
344 #ifdef MULTIPROCESSOR
345 	atomic_or_ulong(&mdpg->mdpg_attrs, set_attributes);
346 #else
347 	mdpg->mdpg_attrs |= set_attributes;
348 #endif
349 }
350 
351 static void
352 pmap_page_syncicache(struct vm_page *pg)
353 {
354 #ifndef MULTIPROCESSOR
355 	struct pmap * const curpmap = curlwp->l_proc->p_vmspace->vm_map.pmap;
356 #endif
357 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
358 	pv_entry_t pv = &mdpg->mdpg_first;
359 	kcpuset_t *onproc;
360 #ifdef MULTIPROCESSOR
361 	kcpuset_create(&onproc, true);
362 	KASSERT(onproc != NULL);
363 #else
364 	onproc = NULL;
365 #endif
366 	VM_PAGEMD_PVLIST_READLOCK(mdpg);
367 	pmap_pvlist_check(mdpg);
368 
369 	if (pv->pv_pmap != NULL) {
370 		for (; pv != NULL; pv = pv->pv_next) {
371 #ifdef MULTIPROCESSOR
372 			kcpuset_merge(onproc, pv->pv_pmap->pm_onproc);
373 			if (kcpuset_match(onproc, kcpuset_running)) {
374 				break;
375 			}
376 #else
377 			if (pv->pv_pmap == curpmap) {
378 				onproc = curcpu()->ci_data.cpu_kcpuset;
379 				break;
380 			}
381 #endif
382 		}
383 	}
384 	pmap_pvlist_check(mdpg);
385 	VM_PAGEMD_PVLIST_UNLOCK(mdpg);
386 	kpreempt_disable();
387 	pmap_md_page_syncicache(pg, onproc);
388 	kpreempt_enable();
389 #ifdef MULTIPROCESSOR
390 	kcpuset_destroy(onproc);
391 #endif
392 }
393 
394 /*
395  * Define the initial bounds of the kernel virtual address space.
396  */
397 void
398 pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp)
399 {
400 
401 	*vstartp = pmap_limits.virtual_start;
402 	*vendp = pmap_limits.virtual_end;
403 }
404 
405 vaddr_t
406 pmap_growkernel(vaddr_t maxkvaddr)
407 {
408 	vaddr_t virtual_end = pmap_limits.virtual_end;
409 	maxkvaddr = pmap_round_seg(maxkvaddr) - 1;
410 
411 	/*
412 	 * Reserve PTEs for the new KVA space.
413 	 */
414 	for (; virtual_end < maxkvaddr; virtual_end += NBSEG) {
415 		pmap_pte_reserve(pmap_kernel(), virtual_end, 0);
416 	}
417 
418 	/*
419 	 * Don't exceed VM_MAX_KERNEL_ADDRESS!
420 	 */
421 	if (virtual_end == 0 || virtual_end > VM_MAX_KERNEL_ADDRESS)
422 		virtual_end = VM_MAX_KERNEL_ADDRESS;
423 
424 	/*
425 	 * Update new end.
426 	 */
427 	pmap_limits.virtual_end = virtual_end;
428 	return virtual_end;
429 }
430 
431 /*
432  * Bootstrap memory allocator (alternative to vm_bootstrap_steal_memory()).
433  * This function allows for early dynamic memory allocation until the virtual
434  * memory system has been bootstrapped.  After that point, either kmem_alloc
435  * or malloc should be used.  This function works by stealing pages from the
436  * (to be) managed page pool, then implicitly mapping the pages (by using
437  * their k0seg addresses) and zeroing them.
438  *
439  * It may be used once the physical memory segments have been pre-loaded
440  * into the vm_physmem[] array.  Early memory allocation MUST use this
441  * interface!  This cannot be used after vm_page_startup(), and will
442  * generate a panic if tried.
443  *
444  * Note that this memory will never be freed, and in essence it is wired
445  * down.
446  *
447  * We must adjust *vstartp and/or *vendp iff we use address space
448  * from the kernel virtual address range defined by pmap_virtual_space().
449  */
450 vaddr_t
451 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp)
452 {
453 	size_t npgs;
454 	paddr_t pa;
455 	vaddr_t va;
456 
457 	uvm_physseg_t maybe_bank = UVM_PHYSSEG_TYPE_INVALID;
458 
459 	size = round_page(size);
460 	npgs = atop(size);
461 
462 	aprint_debug("%s: need %zu pages\n", __func__, npgs);
463 
464 	for (uvm_physseg_t bank = uvm_physseg_get_first();
465 	     uvm_physseg_valid_p(bank);
466 	     bank = uvm_physseg_get_next(bank)) {
467 
468 		if (uvm.page_init_done == true)
469 			panic("pmap_steal_memory: called _after_ bootstrap");
470 
471 		aprint_debug("%s: seg %"PRIxPHYSSEG": %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR"\n",
472 		    __func__, bank,
473 		    uvm_physseg_get_avail_start(bank), uvm_physseg_get_start(bank),
474 		    uvm_physseg_get_avail_end(bank), uvm_physseg_get_end(bank));
475 
476 		if (uvm_physseg_get_avail_start(bank) != uvm_physseg_get_start(bank)
477 		    || uvm_physseg_get_avail_start(bank) >= uvm_physseg_get_avail_end(bank)) {
478 			aprint_debug("%s: seg %"PRIxPHYSSEG": bad start\n", __func__, bank);
479 			continue;
480 		}
481 
482 		if (uvm_physseg_get_avail_end(bank) - uvm_physseg_get_avail_start(bank) < npgs) {
483 			aprint_debug("%s: seg %"PRIxPHYSSEG": too small for %zu pages\n",
484 			    __func__, bank, npgs);
485 			continue;
486 		}
487 
488 		if (!pmap_md_ok_to_steal_p(bank, npgs)) {
489 			continue;
490 		}
491 
492 		/*
493 		 * Always try to allocate from the segment with the least
494 		 * amount of space left.
495 		 */
496 #define VM_PHYSMEM_SPACE(b)	((uvm_physseg_get_avail_end(b)) - (uvm_physseg_get_avail_start(b)))
497 		if (uvm_physseg_valid_p(maybe_bank) == false
498 		    || VM_PHYSMEM_SPACE(bank) < VM_PHYSMEM_SPACE(maybe_bank)) {
499 			maybe_bank = bank;
500 		}
501 	}
502 
503 	if (uvm_physseg_valid_p(maybe_bank)) {
504 		const uvm_physseg_t bank = maybe_bank;
505 
506 		/*
507 		 * There are enough pages here; steal them!
508 		 */
509 		pa = ptoa(uvm_physseg_get_start(bank));
510 		uvm_physseg_unplug(atop(pa), npgs);
511 
512 		aprint_debug("%s: seg %"PRIxPHYSSEG": %zu pages stolen (%#"PRIxPADDR" left)\n",
513 		    __func__, bank, npgs, VM_PHYSMEM_SPACE(bank));
514 
515 		va = pmap_md_map_poolpage(pa, size);
516 		memset((void *)va, 0, size);
517 		return va;
518 	}
519 
520 	/*
521 	 * If we got here, there was no memory left.
522 	 */
523 	panic("pmap_steal_memory: no memory to steal %zu pages", npgs);
524 }
525 
526 /*
527  *	Initialize the pmap module.
528  *	Called by vm_init, to initialize any structures that the pmap
529  *	system needs to map virtual memory.
530  */
531 void
532 pmap_init(void)
533 {
534 	UVMHIST_INIT_STATIC(pmapexechist, pmapexechistbuf);
535 	UVMHIST_INIT_STATIC(pmaphist, pmaphistbuf);
536 
537 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
538 
539 	/*
540 	 * Initialize the segtab lock.
541 	 */
542 	mutex_init(&pmap_segtab_lock, MUTEX_DEFAULT, IPL_HIGH);
543 
544 	/*
545 	 * Set a low water mark on the pv_entry pool, so that we are
546 	 * more likely to have these around even in extreme memory
547 	 * starvation.
548 	 */
549 	pool_setlowat(&pmap_pv_pool, pmap_pv_lowat);
550 
551 	/*
552 	 * Set the page colormask but allow pmap_md_init to override it.
553 	 */
554 	pmap_page_colormask = ptoa(uvmexp.colormask);
555 
556 	pmap_md_init();
557 
558 	/*
559 	 * Now it is safe to enable pv entry recording.
560 	 */
561 	pmap_initialized = true;
562 }
563 
564 /*
565  *	Create and return a physical map.
566  *
567  *	If the size specified for the map
568  *	is zero, the map is an actual physical
569  *	map, and may be referenced by the
570  *	hardware.
571  *
572  *	If the size specified is non-zero,
573  *	the map will be used in software only, and
574  *	is bounded by that size.
575  */
576 pmap_t
577 pmap_create(void)
578 {
579 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
580 	PMAP_COUNT(create);
581 
582 	pmap_t pmap = pool_get(&pmap_pmap_pool, PR_WAITOK);
583 	memset(pmap, 0, PMAP_SIZE);
584 
585 	KASSERT(pmap->pm_pai[0].pai_link.le_prev == NULL);
586 
587 	pmap->pm_count = 1;
588 	pmap->pm_minaddr = VM_MIN_ADDRESS;
589 	pmap->pm_maxaddr = VM_MAXUSER_ADDRESS;
590 
591 	pmap_segtab_init(pmap);
592 
593 #ifdef MULTIPROCESSOR
594 	kcpuset_create(&pmap->pm_active, true);
595 	kcpuset_create(&pmap->pm_onproc, true);
596 	KASSERT(pmap->pm_active != NULL);
597 	KASSERT(pmap->pm_onproc != NULL);
598 #endif
599 
600 	UVMHIST_LOG(pmaphist, " <-- done (pmap=%p)", pmap, 0, 0, 0);
601 
602 	return pmap;
603 }
604 
605 /*
606  *	Retire the given physical map from service.
607  *	Should only be called if the map contains
608  *	no valid mappings.
609  */
610 void
611 pmap_destroy(pmap_t pmap)
612 {
613 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
614 	UVMHIST_LOG(pmaphist, "(pmap=%p)", pmap, 0, 0, 0);
615 
616 	if (atomic_dec_uint_nv(&pmap->pm_count) > 0) {
617 		PMAP_COUNT(dereference);
618 		UVMHIST_LOG(pmaphist, " <-- done (deref)", 0, 0, 0, 0);
619 		return;
620 	}
621 
622 	PMAP_COUNT(destroy);
623 	KASSERT(pmap->pm_count == 0);
624 	kpreempt_disable();
625 	pmap_md_tlb_miss_lock_enter();
626 	pmap_tlb_asid_release_all(pmap);
627 	pmap_segtab_destroy(pmap, NULL, 0);
628 	pmap_md_tlb_miss_lock_exit();
629 
630 #ifdef MULTIPROCESSOR
631 	kcpuset_destroy(pmap->pm_active);
632 	kcpuset_destroy(pmap->pm_onproc);
633 	pmap->pm_active = NULL;
634 	pmap->pm_onproc = NULL;
635 #endif
636 
637 	pool_put(&pmap_pmap_pool, pmap);
638 	kpreempt_enable();
639 
640 	UVMHIST_LOG(pmaphist, " <-- done (freed)", 0, 0, 0, 0);
641 }
642 
643 /*
644  *	Add a reference to the specified pmap.
645  */
646 void
647 pmap_reference(pmap_t pmap)
648 {
649 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
650 	UVMHIST_LOG(pmaphist, "(pmap=%p)", pmap, 0, 0, 0);
651 	PMAP_COUNT(reference);
652 
653 	if (pmap != NULL) {
654 		atomic_inc_uint(&pmap->pm_count);
655 	}
656 
657 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
658 }
659 
660 /*
661  *	Make a new pmap (vmspace) active for the given process.
662  */
663 void
664 pmap_activate(struct lwp *l)
665 {
666 	pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap;
667 #define LNAME(l) \
668 	((l)->l_name ? (l)->l_name : (l)->l_proc->p_comm)
669 
670 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
671 	UVMHIST_LOG(pmaphist, "(l=%p pmap=%p (%s))", l, pmap, LNAME(l), 0);
672 	PMAP_COUNT(activate);
673 
674 	kpreempt_disable();
675 	pmap_md_tlb_miss_lock_enter();
676 	pmap_tlb_asid_acquire(pmap, l);
677 	if (l == curlwp) {
678 		pmap_segtab_activate(pmap, l);
679 	}
680 	pmap_md_tlb_miss_lock_exit();
681 	kpreempt_enable();
682 
683 	UVMHIST_LOG(pmaphist, " <-- done (%u:%u)", l->l_proc->p_pid, l->l_lid, 0, 0);
684 }
685 
686 /*
687  * Remove this page from all physical maps in which it resides.
688  * Reflects back modify bits to the pager.
689  */
690 void
691 pmap_page_remove(struct vm_page *pg)
692 {
693 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
694 
695 	kpreempt_disable();
696 	VM_PAGEMD_PVLIST_LOCK(mdpg);
697 	pmap_pvlist_check(mdpg);
698 
699 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
700 
701 	UVMHIST_LOG(pmapexechist, "pg %p (pa %#"PRIxPADDR")%s: %s",
702 	    pg, VM_PAGE_TO_PHYS(pg), " [page removed]", "execpage cleared");
703 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
704 	pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE|VM_PAGEMD_UNCACHED);
705 #else
706 	pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE);
707 #endif
708 	PMAP_COUNT(exec_uncached_remove);
709 
710 	pv_entry_t pv = &mdpg->mdpg_first;
711 	if (pv->pv_pmap == NULL) {
712 		VM_PAGEMD_PVLIST_UNLOCK(mdpg);
713 		kpreempt_enable();
714 		UVMHIST_LOG(pmaphist, " <-- done (empty)", 0, 0, 0, 0);
715 		return;
716 	}
717 
718 	pv_entry_t npv;
719 	pv_entry_t pvp = NULL;
720 
721 	for (; pv != NULL; pv = npv) {
722 		npv = pv->pv_next;
723 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
724 		if (pv->pv_va & PV_KENTER) {
725 			UVMHIST_LOG(pmaphist, " pv %p pmap %p va %"
726 			    PRIxVADDR" skip", pv, pv->pv_pmap, pv->pv_va, 0);
727 
728 			KASSERT(pv->pv_pmap == pmap_kernel());
729 
730 			/* Assume no more - it'll get fixed if there are */
731 			pv->pv_next = NULL;
732 
733 			/*
734 			 * pvp is non-null when we already have a PV_KENTER
735 			 * pv in pvh_first; otherwise we haven't seen a
736 			 * PV_KENTER pv and we need to copy this one to
737 			 * pvh_first
738 			 */
739 			if (pvp) {
740 				/*
741 				 * The previous PV_KENTER pv needs to point to
742 				 * this PV_KENTER pv
743 				 */
744 				pvp->pv_next = pv;
745 			} else {
746 				pv_entry_t fpv = &mdpg->mdpg_first;
747 				*fpv = *pv;
748 				KASSERT(fpv->pv_pmap == pmap_kernel());
749 			}
750 			pvp = pv;
751 			continue;
752 		}
753 #endif
754 		const pmap_t pmap = pv->pv_pmap;
755 		vaddr_t va = trunc_page(pv->pv_va);
756 		pt_entry_t * const ptep = pmap_pte_lookup(pmap, va);
757 		KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va,
758 		    pmap_limits.virtual_end);
759 		pt_entry_t pte = *ptep;
760 		UVMHIST_LOG(pmaphist, " pv %p pmap %p va %"PRIxVADDR
761 		    " pte %#"PRIxPTE, pv, pmap, va, pte_value(pte));
762 		if (!pte_valid_p(pte))
763 			continue;
764 		const bool is_kernel_pmap_p = (pmap == pmap_kernel());
765 		if (is_kernel_pmap_p) {
766 			PMAP_COUNT(remove_kernel_pages);
767 		} else {
768 			PMAP_COUNT(remove_user_pages);
769 		}
770 		if (pte_wired_p(pte))
771 			pmap->pm_stats.wired_count--;
772 		pmap->pm_stats.resident_count--;
773 
774 		pmap_md_tlb_miss_lock_enter();
775 		const pt_entry_t npte = pte_nv_entry(is_kernel_pmap_p);
776 		*ptep = npte;
777 		/*
778 		 * Flush the TLB for the given address.
779 		 */
780 		pmap_tlb_invalidate_addr(pmap, va);
781 		pmap_md_tlb_miss_lock_exit();
782 
783 		/*
784 		 * non-null means this is a non-pvh_first pv, so we should
785 		 * free it.
786 		 */
787 		if (pvp) {
788 			KASSERT(pvp->pv_pmap == pmap_kernel());
789 			KASSERT(pvp->pv_next == NULL);
790 			pmap_pv_free(pv);
791 		} else {
792 			pv->pv_pmap = NULL;
793 			pv->pv_next = NULL;
794 		}
795 	}
796 
797 	pmap_pvlist_check(mdpg);
798 	VM_PAGEMD_PVLIST_UNLOCK(mdpg);
799 	kpreempt_enable();
800 
801 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
802 }
803 
804 
805 /*
806  *	Make a previously active pmap (vmspace) inactive.
807  */
808 void
809 pmap_deactivate(struct lwp *l)
810 {
811 	pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap;
812 
813 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
814 	UVMHIST_LOG(pmaphist, "(l=%p pmap=%p (%s))", l, pmap, LNAME(l), 0);
815 	PMAP_COUNT(deactivate);
816 
817 	kpreempt_disable();
818 	KASSERT(l == curlwp || l->l_cpu == curlwp->l_cpu);
819 	pmap_md_tlb_miss_lock_enter();
820 	curcpu()->ci_pmap_user_segtab = PMAP_INVALID_SEGTAB_ADDRESS;
821 #ifdef _LP64
822 	curcpu()->ci_pmap_user_seg0tab = NULL;
823 #endif
824 	pmap_tlb_asid_deactivate(pmap);
825 	pmap_md_tlb_miss_lock_exit();
826 	kpreempt_enable();
827 
828 	UVMHIST_LOG(pmaphist, " <-- done (%u:%u)", l->l_proc->p_pid, l->l_lid, 0, 0);
829 }
830 
831 void
832 pmap_update(struct pmap *pmap)
833 {
834 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
835 	UVMHIST_LOG(pmaphist, "(pmap=%p)", pmap, 0, 0, 0);
836 	PMAP_COUNT(update);
837 
838 	kpreempt_disable();
839 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN)
840 	u_int pending = atomic_swap_uint(&pmap->pm_shootdown_pending, 0);
841 	if (pending && pmap_tlb_shootdown_bystanders(pmap))
842 		PMAP_COUNT(shootdown_ipis);
843 #endif
844 	pmap_md_tlb_miss_lock_enter();
845 #if defined(DEBUG) && !defined(MULTIPROCESSOR)
846 	pmap_tlb_check(pmap, pmap_md_tlb_check_entry);
847 #endif /* DEBUG */
848 
849 	/*
850 	 * If pmap_remove_all was called, we deactivated ourselves and nuked
851 	 * our ASID.  Now we have to reactivate ourselves.
852 	 */
853 	if (__predict_false(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE)) {
854 		pmap->pm_flags ^= PMAP_DEFERRED_ACTIVATE;
855 		pmap_tlb_asid_acquire(pmap, curlwp);
856 		pmap_segtab_activate(pmap, curlwp);
857 	}
858 	pmap_md_tlb_miss_lock_exit();
859 	kpreempt_enable();
860 
861 	UVMHIST_LOG(pmaphist, " <-- done%s",
862 	    (pmap == pmap_kernel()) ? " (kernel)" : "", 0, 0, 0);
863 }
864 
865 /*
866  *	Remove the given range of addresses from the specified map.
867  *
868  *	It is assumed that the start and end are properly
869  *	rounded to the page size.
870  */
871 
872 static bool
873 pmap_pte_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep,
874 	uintptr_t flags)
875 {
876 	const pt_entry_t npte = flags;
877 	const bool is_kernel_pmap_p = (pmap == pmap_kernel());
878 
879 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
880 	UVMHIST_LOG(pmaphist, "(pmap=%p %sva=%#"PRIxVADDR"..%#"PRIxVADDR,
881 	    pmap, (is_kernel_pmap_p ? "(kernel) " : ""), sva, eva);
882 	UVMHIST_LOG(pmaphist, "ptep=%p, flags(npte)=%#"PRIxPTR")",
883 	    ptep, flags, 0, 0);
884 
885 	KASSERT(kpreempt_disabled());
886 
887 	for (; sva < eva; sva += NBPG, ptep++) {
888 		const pt_entry_t pte = *ptep;
889 		if (!pte_valid_p(pte))
890 			continue;
891 		if (is_kernel_pmap_p) {
892 			PMAP_COUNT(remove_kernel_pages);
893 		} else {
894 			PMAP_COUNT(remove_user_pages);
895 		}
896 		if (pte_wired_p(pte))
897 			pmap->pm_stats.wired_count--;
898 		pmap->pm_stats.resident_count--;
899 		struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte));
900 		if (__predict_true(pg != NULL)) {
901 			pmap_remove_pv(pmap, sva, pg, pte_modified_p(pte));
902 		}
903 		pmap_md_tlb_miss_lock_enter();
904 		*ptep = npte;
905 		/*
906 		 * Flush the TLB for the given address.
907 		 */
908 		pmap_tlb_invalidate_addr(pmap, sva);
909 		pmap_md_tlb_miss_lock_exit();
910 	}
911 
912 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
913 
914 	return false;
915 }
916 
917 void
918 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva)
919 {
920 	const bool is_kernel_pmap_p = (pmap == pmap_kernel());
921 	const pt_entry_t npte = pte_nv_entry(is_kernel_pmap_p);
922 
923 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
924 	UVMHIST_LOG(pmaphist, "(pmap=%p, va=%#"PRIxVADDR"..%#"PRIxVADDR")",
925 	    pmap, sva, eva, 0);
926 
927 	if (is_kernel_pmap_p) {
928 		PMAP_COUNT(remove_kernel_calls);
929 	} else {
930 		PMAP_COUNT(remove_user_calls);
931 	}
932 #ifdef PMAP_FAULTINFO
933 	curpcb->pcb_faultinfo.pfi_faultaddr = 0;
934 	curpcb->pcb_faultinfo.pfi_repeats = 0;
935 	curpcb->pcb_faultinfo.pfi_faultpte = NULL;
936 #endif
937 	kpreempt_disable();
938 	pmap_addr_range_check(pmap, sva, eva, __func__);
939 	pmap_pte_process(pmap, sva, eva, pmap_pte_remove, npte);
940 	kpreempt_enable();
941 
942 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
943 }
944 
945 /*
946  *	pmap_page_protect:
947  *
948  *	Lower the permission for all mappings to a given page.
949  */
950 void
951 pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
952 {
953 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
954 	pv_entry_t pv;
955 	vaddr_t va;
956 
957 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
958 	UVMHIST_LOG(pmaphist, "(pg=%p (pa %#"PRIxPADDR") prot=%#x)",
959 	    pg, VM_PAGE_TO_PHYS(pg), prot, 0);
960 	PMAP_COUNT(page_protect);
961 
962 	switch (prot) {
963 	case VM_PROT_READ|VM_PROT_WRITE:
964 	case VM_PROT_ALL:
965 		break;
966 
967 	/* copy_on_write */
968 	case VM_PROT_READ:
969 	case VM_PROT_READ|VM_PROT_EXECUTE:
970 		pv = &mdpg->mdpg_first;
971 		kpreempt_disable();
972 		VM_PAGEMD_PVLIST_READLOCK(mdpg);
973 		pmap_pvlist_check(mdpg);
974 		/*
975 		 * Loop over all current mappings setting/clearing as apropos.
976 		 */
977 		if (pv->pv_pmap != NULL) {
978 			while (pv != NULL) {
979 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
980 				if (pv->pv_va & PV_KENTER) {
981 					pv = pv->pv_next;
982 					continue;
983 				}
984 #endif
985 				const pmap_t pmap = pv->pv_pmap;
986 				va = trunc_page(pv->pv_va);
987 				const uintptr_t gen =
988 				    VM_PAGEMD_PVLIST_UNLOCK(mdpg);
989 				pmap_protect(pmap, va, va + PAGE_SIZE, prot);
990 				KASSERT(pv->pv_pmap == pmap);
991 				pmap_update(pmap);
992 				if (gen != VM_PAGEMD_PVLIST_READLOCK(mdpg)) {
993 					pv = &mdpg->mdpg_first;
994 				} else {
995 					pv = pv->pv_next;
996 				}
997 				pmap_pvlist_check(mdpg);
998 			}
999 		}
1000 		pmap_pvlist_check(mdpg);
1001 		VM_PAGEMD_PVLIST_UNLOCK(mdpg);
1002 		kpreempt_enable();
1003 		break;
1004 
1005 	/* remove_all */
1006 	default:
1007 		pmap_page_remove(pg);
1008 	}
1009 
1010 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1011 }
1012 
1013 static bool
1014 pmap_pte_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep,
1015 	uintptr_t flags)
1016 {
1017 	const vm_prot_t prot = (flags & VM_PROT_ALL);
1018 
1019 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1020 	UVMHIST_LOG(pmaphist, "(pmap=%p %sva=%#"PRIxVADDR"..%#"PRIxVADDR,
1021 	    pmap, (pmap == pmap_kernel() ? "(kernel) " : ""), sva, eva);
1022 	UVMHIST_LOG(pmaphist, "ptep=%p, flags(npte)=%#"PRIxPTR")",
1023 	    ptep, flags, 0, 0);
1024 
1025 	KASSERT(kpreempt_disabled());
1026 	/*
1027 	 * Change protection on every valid mapping within this segment.
1028 	 */
1029 	for (; sva < eva; sva += NBPG, ptep++) {
1030 		pt_entry_t pte = *ptep;
1031 		if (!pte_valid_p(pte))
1032 			continue;
1033 		struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte));
1034 		if (pg != NULL && pte_modified_p(pte)) {
1035 			struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
1036 			if (VM_PAGEMD_EXECPAGE_P(mdpg)) {
1037 				KASSERT(mdpg->mdpg_first.pv_pmap != NULL);
1038 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1039 				if (VM_PAGEMD_CACHED_P(mdpg)) {
1040 #endif
1041 					UVMHIST_LOG(pmapexechist,
1042 					    "pg %p (pa %#"PRIxPADDR"): %s",
1043 					    pg, VM_PAGE_TO_PHYS(pg),
1044 					    "syncicached performed", 0);
1045 					pmap_page_syncicache(pg);
1046 					PMAP_COUNT(exec_synced_protect);
1047 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1048 				}
1049 #endif
1050 			}
1051 		}
1052 		pte = pte_prot_downgrade(pte, prot);
1053 		if (*ptep != pte) {
1054 			pmap_md_tlb_miss_lock_enter();
1055 			*ptep = pte;
1056 			/*
1057 			 * Update the TLB if needed.
1058 			 */
1059 			pmap_tlb_update_addr(pmap, sva, pte, PMAP_TLB_NEED_IPI);
1060 			pmap_md_tlb_miss_lock_exit();
1061 		}
1062 	}
1063 
1064 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1065 
1066 	return false;
1067 }
1068 
1069 /*
1070  *	Set the physical protection on the
1071  *	specified range of this map as requested.
1072  */
1073 void
1074 pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
1075 {
1076 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1077 	UVMHIST_LOG(pmaphist,
1078 	    "(pmap=%p, va=%#"PRIxVADDR"..%#"PRIxVADDR", prot=%u)",
1079 	    pmap, sva, eva, prot);
1080 	PMAP_COUNT(protect);
1081 
1082 	if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
1083 		pmap_remove(pmap, sva, eva);
1084 		UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1085 		return;
1086 	}
1087 
1088 	/*
1089 	 * Change protection on every valid mapping within this segment.
1090 	 */
1091 	kpreempt_disable();
1092 	pmap_addr_range_check(pmap, sva, eva, __func__);
1093 	pmap_pte_process(pmap, sva, eva, pmap_pte_protect, prot);
1094 	kpreempt_enable();
1095 
1096 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1097 }
1098 
1099 #if defined(PMAP_VIRTUAL_CACHE_ALIASES) && !defined(PMAP_NO_PV_UNCACHED)
1100 /*
1101  *	pmap_page_cache:
1102  *
1103  *	Change all mappings of a managed page to cached/uncached.
1104  */
1105 void
1106 pmap_page_cache(struct vm_page *pg, bool cached)
1107 {
1108 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
1109 
1110 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1111 	UVMHIST_LOG(pmaphist, "(pg=%p (pa %#"PRIxPADDR") cached=%s)",
1112 	    pg, VM_PAGE_TO_PHYS(pg), cached ? "true" : "false", 0);
1113 
1114 	KASSERT(kpreempt_disabled());
1115 	KASSERT(VM_PAGEMD_PVLIST_LOCKED_P(mdpg));
1116 
1117 	if (cached) {
1118 		pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED);
1119 		PMAP_COUNT(page_cache_restorations);
1120 	} else {
1121 		pmap_page_set_attributes(mdpg, VM_PAGEMD_UNCACHED);
1122 		PMAP_COUNT(page_cache_evictions);
1123 	}
1124 
1125 	for (pv_entry_t pv = &mdpg->mdpg_first; pv != NULL; pv = pv->pv_next) {
1126 		pmap_t pmap = pv->pv_pmap;
1127 		vaddr_t va = trunc_page(pv->pv_va);
1128 
1129 		KASSERT(pmap != NULL);
1130 		KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va));
1131 		pt_entry_t * const ptep = pmap_pte_lookup(pmap, va);
1132 		if (ptep == NULL)
1133 			continue;
1134 		pt_entry_t pte = *ptep;
1135 		if (pte_valid_p(pte)) {
1136 			pte = pte_cached_change(pte, cached);
1137 			pmap_md_tlb_miss_lock_enter();
1138 			*ptep = pte;
1139 			pmap_tlb_update_addr(pmap, va, pte, PMAP_TLB_NEED_IPI);
1140 			pmap_md_tlb_miss_lock_exit();
1141 		}
1142 	}
1143 
1144 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1145 }
1146 #endif	/* PMAP_VIRTUAL_CACHE_ALIASES && !PMAP_NO_PV_UNCACHED */
1147 
1148 /*
1149  *	Insert the given physical page (p) at
1150  *	the specified virtual address (v) in the
1151  *	target physical map with the protection requested.
1152  *
1153  *	If specified, the page will be wired down, meaning
1154  *	that the related pte can not be reclaimed.
1155  *
1156  *	NB:  This is the only routine which MAY NOT lazy-evaluate
1157  *	or lose information.  That is, this routine must actually
1158  *	insert this page into the given map NOW.
1159  */
1160 int
1161 pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
1162 {
1163 	const bool wired = (flags & PMAP_WIRED) != 0;
1164 	const bool is_kernel_pmap_p = (pmap == pmap_kernel());
1165 	u_int update_flags = (flags & VM_PROT_ALL) != 0 ? PMAP_TLB_INSERT : 0;
1166 #ifdef UVMHIST
1167 	struct kern_history * const histp =
1168 	    ((prot & VM_PROT_EXECUTE) ? &pmapexechist : &pmaphist);
1169 #endif
1170 
1171 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(*histp);
1172 #define VM_PROT_STRING(prot) \
1173 	&"\0     " \
1174 	 "(R)\0  " \
1175 	 "(W)\0  " \
1176 	 "(RW)\0 " \
1177 	 "(X)\0  " \
1178 	 "(RX)\0 " \
1179 	 "(WX)\0 " \
1180 	 "(RWX)\0"[UVM_PROTECTION(prot)*6]
1181 	UVMHIST_LOG(*histp, "(pmap=%p, va=%#"PRIxVADDR", pa=%#"PRIxPADDR,
1182 	    pmap, va, pa, 0);
1183 	UVMHIST_LOG(*histp, "prot=%#x%s flags=%#x%s)",
1184 	    prot, VM_PROT_STRING(prot), flags, VM_PROT_STRING(flags));
1185 
1186 	const bool good_color = PMAP_PAGE_COLOROK_P(pa, va);
1187 	if (is_kernel_pmap_p) {
1188 		PMAP_COUNT(kernel_mappings);
1189 		if (!good_color)
1190 			PMAP_COUNT(kernel_mappings_bad);
1191 	} else {
1192 		PMAP_COUNT(user_mappings);
1193 		if (!good_color)
1194 			PMAP_COUNT(user_mappings_bad);
1195 	}
1196 	pmap_addr_range_check(pmap, va, va, __func__);
1197 
1198 	KASSERTMSG(prot & VM_PROT_READ, "no READ (%#x) in prot %#x",
1199 	    VM_PROT_READ, prot);
1200 
1201 	struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
1202 	struct vm_page_md * const mdpg = (pg ? VM_PAGE_TO_MD(pg) : NULL);
1203 
1204 	if (pg) {
1205 		/* Set page referenced/modified status based on flags */
1206 		if (flags & VM_PROT_WRITE) {
1207 			pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED|VM_PAGEMD_REFERENCED);
1208 		} else if (flags & VM_PROT_ALL) {
1209 			pmap_page_set_attributes(mdpg, VM_PAGEMD_REFERENCED);
1210 		}
1211 
1212 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1213 		if (!VM_PAGEMD_CACHED_P(mdpg)) {
1214 			flags |= PMAP_NOCACHE;
1215 			PMAP_COUNT(uncached_mappings);
1216 		}
1217 #endif
1218 
1219 		PMAP_COUNT(managed_mappings);
1220 	} else {
1221 		/*
1222 		 * Assumption: if it is not part of our managed memory
1223 		 * then it must be device memory which may be volatile.
1224 		 */
1225 		if ((flags & PMAP_CACHE_MASK) == 0)
1226 			flags |= PMAP_NOCACHE;
1227 		PMAP_COUNT(unmanaged_mappings);
1228 	}
1229 
1230 	pt_entry_t npte = pte_make_enter(pa, mdpg, prot, flags,
1231 	    is_kernel_pmap_p);
1232 
1233 	kpreempt_disable();
1234 
1235 	pt_entry_t * const ptep = pmap_pte_reserve(pmap, va, flags);
1236 	if (__predict_false(ptep == NULL)) {
1237 		kpreempt_enable();
1238 		UVMHIST_LOG(*histp, " <-- ENOMEM", 0, 0, 0, 0);
1239 		return ENOMEM;
1240 	}
1241 	const pt_entry_t opte = *ptep;
1242 	const bool resident = pte_valid_p(opte);
1243 	bool remap = false;
1244 	if (resident) {
1245 		if (pte_to_paddr(opte) != pa) {
1246 			KASSERT(!is_kernel_pmap_p);
1247 		    	const pt_entry_t rpte = pte_nv_entry(false);
1248 
1249 			pmap_addr_range_check(pmap, va, va + NBPG, __func__);
1250 			pmap_pte_process(pmap, va, va + NBPG, pmap_pte_remove,
1251 			    rpte);
1252 			PMAP_COUNT(user_mappings_changed);
1253 			remap = true;
1254 		}
1255 		update_flags |= PMAP_TLB_NEED_IPI;
1256 	}
1257 
1258 	if (!resident || remap) {
1259 		pmap->pm_stats.resident_count++;
1260 	}
1261 
1262 	/* Done after case that may sleep/return. */
1263 	if (pg)
1264 		pmap_enter_pv(pmap, va, pg, &npte, 0);
1265 
1266 	/*
1267 	 * Now validate mapping with desired protection/wiring.
1268 	 * Assume uniform modified and referenced status for all
1269 	 * MIPS pages in a MACH page.
1270 	 */
1271 	if (wired) {
1272 		pmap->pm_stats.wired_count++;
1273 		npte = pte_wire_entry(npte);
1274 	}
1275 
1276 	UVMHIST_LOG(*histp, "new pte %#"PRIxPTE" (pa %#"PRIxPADDR")",
1277 	    pte_value(npte), pa, 0, 0);
1278 
1279 	KASSERT(pte_valid_p(npte));
1280 
1281 	pmap_md_tlb_miss_lock_enter();
1282 	*ptep = npte;
1283 	pmap_tlb_update_addr(pmap, va, npte, update_flags);
1284 	pmap_md_tlb_miss_lock_exit();
1285 	kpreempt_enable();
1286 
1287 	if (pg != NULL && (prot == (VM_PROT_READ | VM_PROT_EXECUTE))) {
1288 		KASSERT(mdpg != NULL);
1289 		PMAP_COUNT(exec_mappings);
1290 		if (!VM_PAGEMD_EXECPAGE_P(mdpg) && pte_cached_p(npte)) {
1291 			if (!pte_deferred_exec_p(npte)) {
1292 				UVMHIST_LOG(*histp,
1293 				    "va=%#"PRIxVADDR" pg %p: %s syncicache%s",
1294 				    va, pg, "immediate", "");
1295 				pmap_page_syncicache(pg);
1296 				pmap_page_set_attributes(mdpg,
1297 				    VM_PAGEMD_EXECPAGE);
1298 				PMAP_COUNT(exec_synced_mappings);
1299 			} else {
1300 				UVMHIST_LOG(*histp, "va=%#"PRIxVADDR
1301 				    " pg %p: %s syncicache: pte %#x",
1302 				    va, pg, "defer", npte);
1303 			}
1304 		} else {
1305 			UVMHIST_LOG(*histp,
1306 			    "va=%#"PRIxVADDR" pg %p: %s syncicache%s",
1307 			    va, pg, "no",
1308 			    (pte_cached_p(npte)
1309 				? " (already exec)"
1310 				: " (uncached)"));
1311 		}
1312 	} else if (pg != NULL && (prot & VM_PROT_EXECUTE)) {
1313 		KASSERT(mdpg != NULL);
1314 		KASSERT(prot & VM_PROT_WRITE);
1315 		PMAP_COUNT(exec_mappings);
1316 		pmap_page_syncicache(pg);
1317 		pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE);
1318 		UVMHIST_LOG(*histp,
1319 		    "va=%#"PRIxVADDR" pg %p: %s syncicache%s",
1320 		    va, pg, "immediate", " (writeable)");
1321 	}
1322 
1323 	UVMHIST_LOG(*histp, " <-- 0 (OK)", 0, 0, 0, 0);
1324 	return 0;
1325 }
1326 
1327 void
1328 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
1329 {
1330 	pmap_t pmap = pmap_kernel();
1331 	struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
1332 	struct vm_page_md * const mdpg = (pg ? VM_PAGE_TO_MD(pg) : NULL);
1333 
1334 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1335 	UVMHIST_LOG(pmaphist,
1336 	    "(va=%#"PRIxVADDR", pa=%#"PRIxPADDR", prot=%u, flags=%#x)",
1337 	    va, pa, prot, flags);
1338 	PMAP_COUNT(kenter_pa);
1339 
1340 	if (mdpg == NULL) {
1341 		PMAP_COUNT(kenter_pa_unmanaged);
1342 		if ((flags & PMAP_CACHE_MASK) == 0)
1343 			flags |= PMAP_NOCACHE;
1344 	} else {
1345 		if ((flags & PMAP_NOCACHE) == 0 && !PMAP_PAGE_COLOROK_P(pa, va))
1346 			PMAP_COUNT(kenter_pa_bad);
1347 	}
1348 
1349 	pt_entry_t npte = pte_make_kenter_pa(pa, mdpg, prot, flags);
1350 	kpreempt_disable();
1351 	pt_entry_t * const ptep = pmap_pte_lookup(pmap, va);
1352 	KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va,
1353 	    pmap_limits.virtual_end);
1354 	KASSERT(!pte_valid_p(*ptep));
1355 
1356 	/*
1357 	 * No need to track non-managed pages or PMAP_KMPAGEs pages for aliases
1358 	 */
1359 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1360 	if (pg != NULL && (flags & PMAP_KMPAGE) == 0
1361 	    && pmap_md_virtual_cache_aliasing_p()) {
1362 		pmap_enter_pv(pmap, va, pg, &npte, PV_KENTER);
1363 	}
1364 #endif
1365 
1366 	/*
1367 	 * We have the option to force this mapping into the TLB but we
1368 	 * don't.  Instead let the next reference to the page do it.
1369 	 */
1370 	pmap_md_tlb_miss_lock_enter();
1371 	*ptep = npte;
1372 	pmap_tlb_update_addr(pmap_kernel(), va, npte, 0);
1373 	pmap_md_tlb_miss_lock_exit();
1374 	kpreempt_enable();
1375 #if DEBUG > 1
1376 	for (u_int i = 0; i < PAGE_SIZE / sizeof(long); i++) {
1377 		if (((long *)va)[i] != ((long *)pa)[i])
1378 			panic("%s: contents (%lx) of va %#"PRIxVADDR
1379 			    " != contents (%lx) of pa %#"PRIxPADDR, __func__,
1380 			    ((long *)va)[i], va, ((long *)pa)[i], pa);
1381 	}
1382 #endif
1383 
1384 	UVMHIST_LOG(pmaphist, " <-- done (ptep=%p)", ptep, 0, 0, 0);
1385 }
1386 
1387 /*
1388  *	Remove the given range of addresses from the kernel map.
1389  *
1390  *	It is assumed that the start and end are properly
1391  *	rounded to the page size.
1392  */
1393 
1394 static bool
1395 pmap_pte_kremove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep,
1396 	uintptr_t flags)
1397 {
1398 	const pt_entry_t new_pte = pte_nv_entry(true);
1399 
1400 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1401 	UVMHIST_LOG(pmaphist,
1402 	    "(pmap=%p, sva=%#"PRIxVADDR", eva=%#"PRIxVADDR", ptep=%p)",
1403 	    pmap, sva, eva, ptep);
1404 
1405 	KASSERT(kpreempt_disabled());
1406 
1407 	for (; sva < eva; sva += NBPG, ptep++) {
1408 		pt_entry_t pte = *ptep;
1409 		if (!pte_valid_p(pte))
1410 			continue;
1411 
1412 		PMAP_COUNT(kremove_pages);
1413 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1414 		struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte));
1415 		if (pg != NULL && pmap_md_virtual_cache_aliasing_p()) {
1416 			pmap_remove_pv(pmap, sva, pg, !pte_readonly_p(pte));
1417 		}
1418 #endif
1419 
1420 		pmap_md_tlb_miss_lock_enter();
1421 		*ptep = new_pte;
1422 		pmap_tlb_invalidate_addr(pmap, sva);
1423 		pmap_md_tlb_miss_lock_exit();
1424 	}
1425 
1426 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1427 
1428 	return false;
1429 }
1430 
1431 void
1432 pmap_kremove(vaddr_t va, vsize_t len)
1433 {
1434 	const vaddr_t sva = trunc_page(va);
1435 	const vaddr_t eva = round_page(va + len);
1436 
1437 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1438 	UVMHIST_LOG(pmaphist, "(va=%#"PRIxVADDR", len=%#"PRIxVSIZE")",
1439 	    va, len, 0, 0);
1440 
1441 	kpreempt_disable();
1442 	pmap_pte_process(pmap_kernel(), sva, eva, pmap_pte_kremove, 0);
1443 	kpreempt_enable();
1444 
1445 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1446 }
1447 
1448 void
1449 pmap_remove_all(struct pmap *pmap)
1450 {
1451 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1452 	UVMHIST_LOG(pmaphist, "(pm=%p)", pmap, 0, 0, 0);
1453 
1454 	KASSERT(pmap != pmap_kernel());
1455 
1456 	kpreempt_disable();
1457 	/*
1458 	 * Free all of our ASIDs which means we can skip doing all the
1459 	 * tlb_invalidate_addrs().
1460 	 */
1461 	pmap_md_tlb_miss_lock_enter();
1462 #ifdef MULTIPROCESSOR
1463 	// This should be the last CPU with this pmap onproc
1464 	KASSERT(!kcpuset_isotherset(pmap->pm_onproc, cpu_index(curcpu())));
1465 	if (kcpuset_isset(pmap->pm_onproc, cpu_index(curcpu())))
1466 #endif
1467 		pmap_tlb_asid_deactivate(pmap);
1468 #ifdef MULTIPROCESSOR
1469 	KASSERT(kcpuset_iszero(pmap->pm_onproc));
1470 #endif
1471 	pmap_tlb_asid_release_all(pmap);
1472 	pmap_md_tlb_miss_lock_exit();
1473 	pmap->pm_flags |= PMAP_DEFERRED_ACTIVATE;
1474 
1475 #ifdef PMAP_FAULTINFO
1476 	curpcb->pcb_faultinfo.pfi_faultaddr = 0;
1477 	curpcb->pcb_faultinfo.pfi_repeats = 0;
1478 	curpcb->pcb_faultinfo.pfi_faultpte = NULL;
1479 #endif
1480 	kpreempt_enable();
1481 
1482 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1483 }
1484 
1485 /*
1486  *	Routine:	pmap_unwire
1487  *	Function:	Clear the wired attribute for a map/virtual-address
1488  *			pair.
1489  *	In/out conditions:
1490  *			The mapping must already exist in the pmap.
1491  */
1492 void
1493 pmap_unwire(pmap_t pmap, vaddr_t va)
1494 {
1495 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1496 	UVMHIST_LOG(pmaphist, "(pmap=%p, va=%#"PRIxVADDR")", pmap, va, 0, 0);
1497 	PMAP_COUNT(unwire);
1498 
1499 	/*
1500 	 * Don't need to flush the TLB since PG_WIRED is only in software.
1501 	 */
1502 	kpreempt_disable();
1503 	pmap_addr_range_check(pmap, va, va, __func__);
1504 	pt_entry_t * const ptep = pmap_pte_lookup(pmap, va);
1505 	KASSERTMSG(ptep != NULL, "pmap %p va %#"PRIxVADDR" invalid STE",
1506 	    pmap, va);
1507 	pt_entry_t pte = *ptep;
1508 	KASSERTMSG(pte_valid_p(pte),
1509 	    "pmap %p va %#"PRIxVADDR" invalid PTE %#"PRIxPTE" @ %p",
1510 	    pmap, va, pte_value(pte), ptep);
1511 
1512 	if (pte_wired_p(pte)) {
1513 		pmap_md_tlb_miss_lock_enter();
1514 		*ptep = pte_unwire_entry(pte);
1515 		pmap_md_tlb_miss_lock_exit();
1516 		pmap->pm_stats.wired_count--;
1517 	}
1518 #ifdef DIAGNOSTIC
1519 	else {
1520 		printf("%s: wiring for pmap %p va %#"PRIxVADDR" unchanged!\n",
1521 		    __func__, pmap, va);
1522 	}
1523 #endif
1524 	kpreempt_enable();
1525 
1526 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1527 }
1528 
1529 /*
1530  *	Routine:	pmap_extract
1531  *	Function:
1532  *		Extract the physical page address associated
1533  *		with the given map/virtual_address pair.
1534  */
1535 bool
1536 pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap)
1537 {
1538 	paddr_t pa;
1539 
1540 	if (pmap == pmap_kernel()) {
1541 		if (pmap_md_direct_mapped_vaddr_p(va)) {
1542 			pa = pmap_md_direct_mapped_vaddr_to_paddr(va);
1543 			goto done;
1544 		}
1545 		if (pmap_md_io_vaddr_p(va))
1546 			panic("pmap_extract: io address %#"PRIxVADDR"", va);
1547 
1548 		if (va >= pmap_limits.virtual_end)
1549 			panic("%s: illegal kernel mapped address %#"PRIxVADDR,
1550 			    __func__, va);
1551 	}
1552 	kpreempt_disable();
1553 	const pt_entry_t * const ptep = pmap_pte_lookup(pmap, va);
1554 	if (ptep == NULL || !pte_valid_p(*ptep)) {
1555 		kpreempt_enable();
1556 		return false;
1557 	}
1558 	pa = pte_to_paddr(*ptep) | (va & PGOFSET);
1559 	kpreempt_enable();
1560 done:
1561 	if (pap != NULL) {
1562 		*pap = pa;
1563 	}
1564 	return true;
1565 }
1566 
1567 /*
1568  *	Copy the range specified by src_addr/len
1569  *	from the source map to the range dst_addr/len
1570  *	in the destination map.
1571  *
1572  *	This routine is only advisory and need not do anything.
1573  */
1574 void
1575 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr, vsize_t len,
1576     vaddr_t src_addr)
1577 {
1578 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1579 	PMAP_COUNT(copy);
1580 }
1581 
1582 /*
1583  *	pmap_clear_reference:
1584  *
1585  *	Clear the reference bit on the specified physical page.
1586  */
1587 bool
1588 pmap_clear_reference(struct vm_page *pg)
1589 {
1590 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
1591 
1592 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1593 	UVMHIST_LOG(pmaphist, "(pg=%p (pa %#"PRIxPADDR"))",
1594 	   pg, VM_PAGE_TO_PHYS(pg), 0,0);
1595 
1596 	bool rv = pmap_page_clear_attributes(mdpg, VM_PAGEMD_REFERENCED);
1597 
1598 	UVMHIST_LOG(pmaphist, " <-- %s", rv ? "true" : "false", 0, 0, 0);
1599 
1600 	return rv;
1601 }
1602 
1603 /*
1604  *	pmap_is_referenced:
1605  *
1606  *	Return whether or not the specified physical page is referenced
1607  *	by any physical maps.
1608  */
1609 bool
1610 pmap_is_referenced(struct vm_page *pg)
1611 {
1612 	return VM_PAGEMD_REFERENCED_P(VM_PAGE_TO_MD(pg));
1613 }
1614 
1615 /*
1616  *	Clear the modify bits on the specified physical page.
1617  */
1618 bool
1619 pmap_clear_modify(struct vm_page *pg)
1620 {
1621 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
1622 	pv_entry_t pv = &mdpg->mdpg_first;
1623 	pv_entry_t pv_next;
1624 
1625 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1626 	UVMHIST_LOG(pmaphist, "(pg=%p (%#"PRIxPADDR"))",
1627 	    pg, VM_PAGE_TO_PHYS(pg), 0,0);
1628 	PMAP_COUNT(clear_modify);
1629 
1630 	if (VM_PAGEMD_EXECPAGE_P(mdpg)) {
1631 		if (pv->pv_pmap == NULL) {
1632 			UVMHIST_LOG(pmapexechist,
1633 			    "pg %p (pa %#"PRIxPADDR"): %s",
1634 			    pg, VM_PAGE_TO_PHYS(pg), "execpage cleared", 0);
1635 			pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE);
1636 			PMAP_COUNT(exec_uncached_clear_modify);
1637 		} else {
1638 			UVMHIST_LOG(pmapexechist,
1639 			    "pg %p (pa %#"PRIxPADDR"): %s",
1640 			    pg, VM_PAGE_TO_PHYS(pg), "syncicache performed", 0);
1641 			pmap_page_syncicache(pg);
1642 			PMAP_COUNT(exec_synced_clear_modify);
1643 		}
1644 	}
1645 	if (!pmap_page_clear_attributes(mdpg, VM_PAGEMD_MODIFIED)) {
1646 		UVMHIST_LOG(pmaphist, " <-- false", 0, 0, 0, 0);
1647 		return false;
1648 	}
1649 	if (pv->pv_pmap == NULL) {
1650 		UVMHIST_LOG(pmaphist, " <-- true (no mappings)", 0, 0, 0, 0);
1651 		return true;
1652 	}
1653 
1654 	/*
1655 	 * remove write access from any pages that are dirty
1656 	 * so we can tell if they are written to again later.
1657 	 * flush the VAC first if there is one.
1658 	 */
1659 	kpreempt_disable();
1660 	KASSERT(!VM_PAGEMD_PVLIST_LOCKED_P(mdpg));
1661 	VM_PAGEMD_PVLIST_READLOCK(mdpg);
1662 	pmap_pvlist_check(mdpg);
1663 	for (; pv != NULL; pv = pv_next) {
1664 		pmap_t pmap = pv->pv_pmap;
1665 		vaddr_t va = trunc_page(pv->pv_va);
1666 
1667 		pv_next = pv->pv_next;
1668 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1669 		if (pv->pv_va & PV_KENTER)
1670 			continue;
1671 #endif
1672 		pt_entry_t * const ptep = pmap_pte_lookup(pmap, va);
1673 		KASSERT(ptep);
1674 		pt_entry_t pte = pte_prot_nowrite(*ptep);
1675 		if (*ptep == pte) {
1676 			continue;
1677 		}
1678 		KASSERT(pte_valid_p(pte));
1679 		const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg);
1680 		pmap_md_tlb_miss_lock_enter();
1681 		*ptep = pte;
1682 		pmap_tlb_invalidate_addr(pmap, va);
1683 		pmap_md_tlb_miss_lock_exit();
1684 		pmap_update(pmap);
1685 		if (__predict_false(gen != VM_PAGEMD_PVLIST_READLOCK(mdpg))) {
1686 			/*
1687 			 * The list changed!  So restart from the beginning.
1688 			 */
1689 			pv_next = &mdpg->mdpg_first;
1690 			pmap_pvlist_check(mdpg);
1691 		}
1692 	}
1693 	pmap_pvlist_check(mdpg);
1694 	VM_PAGEMD_PVLIST_UNLOCK(mdpg);
1695 	kpreempt_enable();
1696 
1697 	UVMHIST_LOG(pmaphist, " <-- true (mappings changed)", 0, 0, 0, 0);
1698 	return true;
1699 }
1700 
1701 /*
1702  *	pmap_is_modified:
1703  *
1704  *	Return whether or not the specified physical page is modified
1705  *	by any physical maps.
1706  */
1707 bool
1708 pmap_is_modified(struct vm_page *pg)
1709 {
1710 	return VM_PAGEMD_MODIFIED_P(VM_PAGE_TO_MD(pg));
1711 }
1712 
1713 /*
1714  *	pmap_set_modified:
1715  *
1716  *	Sets the page modified reference bit for the specified page.
1717  */
1718 void
1719 pmap_set_modified(paddr_t pa)
1720 {
1721 	struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
1722 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
1723 	pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED|VM_PAGEMD_REFERENCED);
1724 }
1725 
1726 /******************** pv_entry management ********************/
1727 
1728 static void
1729 pmap_pvlist_check(struct vm_page_md *mdpg)
1730 {
1731 #ifdef DEBUG
1732 	pv_entry_t pv = &mdpg->mdpg_first;
1733 	if (pv->pv_pmap != NULL) {
1734 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1735 		const u_int colormask = uvmexp.colormask;
1736 		u_int colors = 0;
1737 #endif
1738 		for (; pv != NULL; pv = pv->pv_next) {
1739 			KASSERT(pv->pv_pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(pv->pv_va));
1740 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1741 			colors |= __BIT(atop(pv->pv_va) & colormask);
1742 #endif
1743 		}
1744 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1745 		// Assert there if there more than 1 color mapped, that they
1746 		// are uncached.
1747 		KASSERTMSG(!pmap_md_virtual_cache_aliasing_p()
1748 		    || colors == 0 || (colors & (colors-1)) == 0
1749 		    || VM_PAGEMD_UNCACHED_P(mdpg), "colors=%#x uncached=%u",
1750 		    colors, VM_PAGEMD_UNCACHED_P(mdpg));
1751 #endif
1752 	}
1753 #endif /* DEBUG */
1754 }
1755 
1756 /*
1757  * Enter the pmap and virtual address into the
1758  * physical to virtual map table.
1759  */
1760 void
1761 pmap_enter_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, pt_entry_t *nptep,
1762     u_int flags)
1763 {
1764 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
1765 	pv_entry_t pv, npv, apv;
1766 #ifdef UVMHIST
1767 	bool first = false;
1768 #endif
1769 
1770 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1771 	UVMHIST_LOG(pmaphist,
1772 	    "(pmap=%p va=%#"PRIxVADDR" pg=%p (%#"PRIxPADDR")",
1773 	    pmap, va, pg, VM_PAGE_TO_PHYS(pg));
1774 	UVMHIST_LOG(pmaphist, "nptep=%p (%#"PRIxPTE"))",
1775 	    nptep, pte_value(*nptep), 0, 0);
1776 
1777 	KASSERT(kpreempt_disabled());
1778 	KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va));
1779 	KASSERTMSG(pmap != pmap_kernel() || !pmap_md_io_vaddr_p(va),
1780 	    "va %#"PRIxVADDR, va);
1781 
1782 	apv = NULL;
1783 	VM_PAGEMD_PVLIST_LOCK(mdpg);
1784 again:
1785 	pv = &mdpg->mdpg_first;
1786 	pmap_pvlist_check(mdpg);
1787 	if (pv->pv_pmap == NULL) {
1788 		KASSERT(pv->pv_next == NULL);
1789 		/*
1790 		 * No entries yet, use header as the first entry
1791 		 */
1792 		PMAP_COUNT(primary_mappings);
1793 		PMAP_COUNT(mappings);
1794 #ifdef UVMHIST
1795 		first = true;
1796 #endif
1797 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1798 		KASSERT(VM_PAGEMD_CACHED_P(mdpg));
1799 		// If the new mapping has an incompatible color the last
1800 		// mapping of this page, clean the page before using it.
1801 		if (!PMAP_PAGE_COLOROK_P(va, pv->pv_va)) {
1802 			pmap_md_vca_clean(pg, PMAP_WBINV);
1803 		}
1804 #endif
1805 		pv->pv_pmap = pmap;
1806 		pv->pv_va = va | flags;
1807 	} else {
1808 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1809 		if (pmap_md_vca_add(pg, va, nptep)) {
1810 			goto again;
1811 		}
1812 #endif
1813 
1814 		/*
1815 		 * There is at least one other VA mapping this page.
1816 		 * Place this entry after the header.
1817 		 *
1818 		 * Note: the entry may already be in the table if
1819 		 * we are only changing the protection bits.
1820 		 */
1821 
1822 #ifdef PARANOIADIAG
1823 		const paddr_t pa = VM_PAGE_TO_PHYS(pg);
1824 #endif
1825 		for (npv = pv; npv; npv = npv->pv_next) {
1826 			if (pmap == npv->pv_pmap
1827 			    && va == trunc_page(npv->pv_va)) {
1828 #ifdef PARANOIADIAG
1829 				pt_entry_t *ptep = pmap_pte_lookup(pmap, va);
1830 				pt_entry_t pte = (ptep != NULL) ? *ptep : 0;
1831 				if (!pte_valid_p(pte) || pte_to_paddr(pte) != pa)
1832 					printf("%s: found va %#"PRIxVADDR
1833 					    " pa %#"PRIxPADDR
1834 					    " in pv_table but != %#"PRIxPTE"\n",
1835 					    __func__, va, pa, pte_value(pte));
1836 #endif
1837 				PMAP_COUNT(remappings);
1838 				VM_PAGEMD_PVLIST_UNLOCK(mdpg);
1839 				if (__predict_false(apv != NULL))
1840 					pmap_pv_free(apv);
1841 
1842 				UVMHIST_LOG(pmaphist, " <-- done pv=%p%s",
1843 				    pv, " (reused)", 0, 0);
1844 				return;
1845 			}
1846 		}
1847 		if (__predict_true(apv == NULL)) {
1848 			/*
1849 			 * To allocate a PV, we have to release the PVLIST lock
1850 			 * so get the page generation.  We allocate the PV, and
1851 			 * then reacquire the lock.
1852 			 */
1853 			pmap_pvlist_check(mdpg);
1854 			const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg);
1855 
1856 			apv = (pv_entry_t)pmap_pv_alloc();
1857 			if (apv == NULL)
1858 				panic("pmap_enter_pv: pmap_pv_alloc() failed");
1859 
1860 			/*
1861 			 * If the generation has changed, then someone else
1862 			 * tinkered with this page so we should start over.
1863 			 */
1864 			if (gen != VM_PAGEMD_PVLIST_LOCK(mdpg))
1865 				goto again;
1866 		}
1867 		npv = apv;
1868 		apv = NULL;
1869 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1870 		/*
1871 		 * If need to deal with virtual cache aliases, keep mappings
1872 		 * in the kernel pmap at the head of the list.  This allows
1873 		 * the VCA code to easily use them for cache operations if
1874 		 * present.
1875 		 */
1876 		pmap_t kpmap = pmap_kernel();
1877 		if (pmap != kpmap) {
1878 			while (pv->pv_pmap == kpmap && pv->pv_next != NULL) {
1879 				pv = pv->pv_next;
1880 			}
1881 		}
1882 #endif
1883 		npv->pv_va = va | flags;
1884 		npv->pv_pmap = pmap;
1885 		npv->pv_next = pv->pv_next;
1886 		pv->pv_next = npv;
1887 		PMAP_COUNT(mappings);
1888 	}
1889 	pmap_pvlist_check(mdpg);
1890 	VM_PAGEMD_PVLIST_UNLOCK(mdpg);
1891 	if (__predict_false(apv != NULL))
1892 		pmap_pv_free(apv);
1893 
1894 	UVMHIST_LOG(pmaphist, " <-- done pv=%p%s",
1895 	    pv, first ? " (first pv)" : "",0,0);
1896 }
1897 
1898 /*
1899  * Remove a physical to virtual address translation.
1900  * If cache was inhibited on this page, and there are no more cache
1901  * conflicts, restore caching.
1902  * Flush the cache if the last page is removed (should always be cached
1903  * at this point).
1904  */
1905 void
1906 pmap_remove_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, bool dirty)
1907 {
1908 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
1909 	pv_entry_t pv, npv;
1910 	bool last;
1911 
1912 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
1913 	UVMHIST_LOG(pmaphist,
1914 	    "(pmap=%p, va=%#"PRIxVADDR", pg=%p (pa %#"PRIxPADDR")",
1915 	    pmap, va, pg, VM_PAGE_TO_PHYS(pg));
1916 	UVMHIST_LOG(pmaphist, "dirty=%s)", dirty ? "true" : "false", 0, 0, 0);
1917 
1918 	KASSERT(kpreempt_disabled());
1919 	KASSERT((va & PAGE_MASK) == 0);
1920 	pv = &mdpg->mdpg_first;
1921 
1922 	VM_PAGEMD_PVLIST_LOCK(mdpg);
1923 	pmap_pvlist_check(mdpg);
1924 
1925 	/*
1926 	 * If it is the first entry on the list, it is actually
1927 	 * in the header and we must copy the following entry up
1928 	 * to the header.  Otherwise we must search the list for
1929 	 * the entry.  In either case we free the now unused entry.
1930 	 */
1931 
1932 	last = false;
1933 	if (pmap == pv->pv_pmap && va == trunc_page(pv->pv_va)) {
1934 		npv = pv->pv_next;
1935 		if (npv) {
1936 			*pv = *npv;
1937 			KASSERT(pv->pv_pmap != NULL);
1938 		} else {
1939 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1940 			pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED);
1941 #endif
1942 			pv->pv_pmap = NULL;
1943 			last = true;	/* Last mapping removed */
1944 		}
1945 		PMAP_COUNT(remove_pvfirst);
1946 	} else {
1947 		for (npv = pv->pv_next; npv; pv = npv, npv = npv->pv_next) {
1948 			PMAP_COUNT(remove_pvsearch);
1949 			if (pmap == npv->pv_pmap && va == trunc_page(npv->pv_va))
1950 				break;
1951 		}
1952 		if (npv) {
1953 			pv->pv_next = npv->pv_next;
1954 		}
1955 	}
1956 
1957 	pmap_pvlist_check(mdpg);
1958 	VM_PAGEMD_PVLIST_UNLOCK(mdpg);
1959 
1960 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
1961 	pmap_md_vca_remove(pg, va, dirty, last);
1962 #endif
1963 
1964 	/*
1965 	 * Free the pv_entry if needed.
1966 	 */
1967 	if (npv)
1968 		pmap_pv_free(npv);
1969 	if (VM_PAGEMD_EXECPAGE_P(mdpg) && dirty) {
1970 		if (last) {
1971 			/*
1972 			 * If this was the page's last mapping, we no longer
1973 			 * care about its execness.
1974 			 */
1975 			UVMHIST_LOG(pmapexechist,
1976 			    "pg %p (pa %#"PRIxPADDR")%s: %s",
1977 			    pg, VM_PAGE_TO_PHYS(pg),
1978 			    last ? " [last mapping]" : "",
1979 			    "execpage cleared");
1980 			pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE);
1981 			PMAP_COUNT(exec_uncached_remove);
1982 		} else {
1983 			/*
1984 			 * Someone still has it mapped as an executable page
1985 			 * so we must sync it.
1986 			 */
1987 			UVMHIST_LOG(pmapexechist,
1988 			    "pg %p (pa %#"PRIxPADDR")%s: %s",
1989 			    pg, VM_PAGE_TO_PHYS(pg),
1990 			    last ? " [last mapping]" : "",
1991 			    "performed syncicache");
1992 			pmap_page_syncicache(pg);
1993 			PMAP_COUNT(exec_synced_remove);
1994 		}
1995 	}
1996 
1997 	UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0);
1998 }
1999 
2000 #if defined(MULTIPROCESSOR)
2001 struct pmap_pvlist_info {
2002 	kmutex_t *pli_locks[PAGE_SIZE / 32];
2003 	volatile u_int pli_lock_refs[PAGE_SIZE / 32];
2004 	volatile u_int pli_lock_index;
2005 	u_int pli_lock_mask;
2006 } pmap_pvlist_info;
2007 
2008 void
2009 pmap_pvlist_lock_init(size_t cache_line_size)
2010 {
2011 	struct pmap_pvlist_info * const pli = &pmap_pvlist_info;
2012 	const vaddr_t lock_page = uvm_pageboot_alloc(PAGE_SIZE);
2013 	vaddr_t lock_va = lock_page;
2014 	if (sizeof(kmutex_t) > cache_line_size) {
2015 		cache_line_size = roundup2(sizeof(kmutex_t), cache_line_size);
2016 	}
2017 	const size_t nlocks = PAGE_SIZE / cache_line_size;
2018 	KASSERT((nlocks & (nlocks - 1)) == 0);
2019 	/*
2020 	 * Now divide the page into a number of mutexes, one per cacheline.
2021 	 */
2022 	for (size_t i = 0; i < nlocks; lock_va += cache_line_size, i++) {
2023 		kmutex_t * const lock = (kmutex_t *)lock_va;
2024 		mutex_init(lock, MUTEX_DEFAULT, IPL_HIGH);
2025 		pli->pli_locks[i] = lock;
2026 	}
2027 	pli->pli_lock_mask = nlocks - 1;
2028 }
2029 
2030 kmutex_t *
2031 pmap_pvlist_lock_addr(struct vm_page_md *mdpg)
2032 {
2033 	struct pmap_pvlist_info * const pli = &pmap_pvlist_info;
2034 	kmutex_t *lock = mdpg->mdpg_lock;
2035 
2036 	/*
2037 	 * Allocate a lock on an as-needed basis.  This will hopefully give us
2038 	 * semi-random distribution not based on page color.
2039 	 */
2040 	if (__predict_false(lock == NULL)) {
2041 		size_t locknum = atomic_add_int_nv(&pli->pli_lock_index, 37);
2042 		size_t lockid = locknum & pli->pli_lock_mask;
2043 		kmutex_t * const new_lock = pli->pli_locks[lockid];
2044 		/*
2045 		 * Set the lock.  If some other thread already did, just use
2046 		 * the one they assigned.
2047 		 */
2048 		lock = atomic_cas_ptr(&mdpg->mdpg_lock, NULL, new_lock);
2049 		if (lock == NULL) {
2050 			lock = new_lock;
2051 			atomic_inc_uint(&pli->pli_lock_refs[lockid]);
2052 		}
2053 	}
2054 
2055 	/*
2056 	 * Now finally provide the lock.
2057 	 */
2058 	return lock;
2059 }
2060 #else /* !MULTIPROCESSOR */
2061 void
2062 pmap_pvlist_lock_init(size_t cache_line_size)
2063 {
2064 	mutex_init(&pmap_pvlist_mutex, MUTEX_DEFAULT, IPL_HIGH);
2065 }
2066 
2067 #ifdef MODULAR
2068 kmutex_t *
2069 pmap_pvlist_lock_addr(struct vm_page_md *mdpg)
2070 {
2071 	/*
2072 	 * We just use a global lock.
2073 	 */
2074 	if (__predict_false(mdpg->mdpg_lock == NULL)) {
2075 		mdpg->mdpg_lock = &pmap_pvlist_mutex;
2076 	}
2077 
2078 	/*
2079 	 * Now finally provide the lock.
2080 	 */
2081 	return mdpg->mdpg_lock;
2082 }
2083 #endif /* MODULAR */
2084 #endif /* !MULTIPROCESSOR */
2085 
2086 /*
2087  * pmap_pv_page_alloc:
2088  *
2089  *	Allocate a page for the pv_entry pool.
2090  */
2091 void *
2092 pmap_pv_page_alloc(struct pool *pp, int flags)
2093 {
2094 	struct vm_page * const pg = PMAP_ALLOC_POOLPAGE(UVM_PGA_USERESERVE);
2095 	if (pg == NULL)
2096 		return NULL;
2097 
2098 	return (void *)pmap_map_poolpage(VM_PAGE_TO_PHYS(pg));
2099 }
2100 
2101 /*
2102  * pmap_pv_page_free:
2103  *
2104  *	Free a pv_entry pool page.
2105  */
2106 void
2107 pmap_pv_page_free(struct pool *pp, void *v)
2108 {
2109 	vaddr_t va = (vaddr_t)v;
2110 
2111 	KASSERT(pmap_md_direct_mapped_vaddr_p(va));
2112 	const paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va);
2113 	struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
2114 	KASSERT(pg != NULL);
2115 #ifdef PMAP_VIRTUAL_CACHE_ALIASES
2116 	kpreempt_disable();
2117 	pmap_md_vca_remove(pg, va, true, true);
2118 	kpreempt_enable();
2119 #endif
2120 	pmap_page_clear_attributes(VM_PAGE_TO_MD(pg), VM_PAGEMD_POOLPAGE);
2121 	uvm_pagefree(pg);
2122 }
2123 
2124 #ifdef PMAP_PREFER
2125 /*
2126  * Find first virtual address >= *vap that doesn't cause
2127  * a cache alias conflict.
2128  */
2129 void
2130 pmap_prefer(vaddr_t foff, vaddr_t *vap, vsize_t sz, int td)
2131 {
2132 	vsize_t prefer_mask = ptoa(uvmexp.colormask);
2133 
2134 	PMAP_COUNT(prefer_requests);
2135 
2136 	prefer_mask |= pmap_md_cache_prefer_mask();
2137 
2138 	if (prefer_mask) {
2139 		vaddr_t	va = *vap;
2140 		vsize_t d = (foff - va) & prefer_mask;
2141 		if (d) {
2142 			if (td)
2143 				*vap = trunc_page(va - ((-d) & prefer_mask));
2144 			else
2145 				*vap = round_page(va + d);
2146 			PMAP_COUNT(prefer_adjustments);
2147 		}
2148 	}
2149 }
2150 #endif /* PMAP_PREFER */
2151 
2152 #ifdef PMAP_MAP_POOLPAGE
2153 vaddr_t
2154 pmap_map_poolpage(paddr_t pa)
2155 {
2156 	struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
2157 	KASSERT(pg);
2158 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
2159 	pmap_page_set_attributes(mdpg, VM_PAGEMD_POOLPAGE);
2160 
2161 	return pmap_md_map_poolpage(pa, NBPG);
2162 }
2163 
2164 paddr_t
2165 pmap_unmap_poolpage(vaddr_t va)
2166 {
2167 	KASSERT(pmap_md_direct_mapped_vaddr_p(va));
2168 	paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va);
2169 
2170 	struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
2171 	KASSERT(pg != NULL);
2172 	pmap_page_clear_attributes(VM_PAGE_TO_MD(pg), VM_PAGEMD_POOLPAGE);
2173 	pmap_md_unmap_poolpage(va, NBPG);
2174 
2175 	return pa;
2176 }
2177 #endif /* PMAP_MAP_POOLPAGE */
2178