xref: /netbsd-src/sys/uvm/pmap/pmap_tlb.c (revision d90047b5d07facf36e6c01dcc0bded8997ce9cc2)
1 /*	$NetBSD: pmap_tlb.c,v 1.33 2020/04/14 05:43:57 skrll Exp $	*/
2 
3 /*-
4  * Copyright (c) 2010 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Matt Thomas at 3am Software Foundry.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 #include <sys/cdefs.h>
33 
34 __KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.33 2020/04/14 05:43:57 skrll Exp $");
35 
36 /*
37  * Manages address spaces in a TLB.
38  *
39  * Normally there is a 1:1 mapping between a TLB and a CPU.  However, some
40  * implementations may share a TLB between multiple CPUs (really CPU thread
41  * contexts).  This requires the TLB abstraction to be separated from the
42  * CPU abstraction.  It also requires that the TLB be locked while doing
43  * TLB activities.
44  *
45  * For each TLB, we track the ASIDs in use in a bitmap and a list of pmaps
46  * that have a valid ASID.
47  *
48  * We allocate ASIDs in increasing order until we have exhausted the supply,
49  * then reinitialize the ASID space, and start allocating again at 1.  When
50  * allocating from the ASID bitmap, we skip any ASID who has a corresponding
51  * bit set in the ASID bitmap.  Eventually this causes the ASID bitmap to fill
52  * and, when completely filled, a reinitialization of the ASID space.
53  *
54  * To reinitialize the ASID space, the ASID bitmap is reset and then the ASIDs
55  * of non-kernel TLB entries get recorded in the ASID bitmap.  If the entries
56  * in TLB consume more than half of the ASID space, all ASIDs are invalidated,
57  * the ASID bitmap is recleared, and the list of pmaps is emptied.  Otherwise,
58  * (the normal case), any ASID present in the TLB (even those which are no
59  * longer used by a pmap) will remain active (allocated) and all other ASIDs
60  * will be freed.  If the size of the TLB is much smaller than the ASID space,
61  * this algorithm completely avoids TLB invalidation.
62  *
63  * For multiprocessors, we also have to deal TLB invalidation requests from
64  * other CPUs, some of which are dealt with the reinitialization of the ASID
65  * space.  Whereas above we keep the ASIDs of those pmaps which have active
66  * TLB entries, this type of reinitialization preserves the ASIDs of any
67  * "onproc" user pmap and all other ASIDs will be freed.  We must do this
68  * since we can't change the current ASID.
69  *
70  * Each pmap has two bitmaps: pm_active and pm_onproc.  Each bit in pm_active
71  * indicates whether that pmap has an allocated ASID for a CPU.  Each bit in
72  * pm_onproc indicates that the pmap's ASID is in use, i.e. a CPU has it in its
73  * "current ASID" field, e.g. the ASID field of the COP 0 register EntryHi for
74  * MIPS, or the ASID field of TTBR0 for AA64.  The bit number used in these
75  * bitmaps comes from the CPU's cpu_index().  Even though these bitmaps contain
76  * the bits for all CPUs, the bits that  correspond to the bits belonging to
77  * the CPUs sharing a TLB can only be manipulated while holding that TLB's
78  * lock.  Atomic ops must be used to update them since multiple CPUs may be
79  * changing different sets of bits at same time but these sets never overlap.
80  *
81  * When a change to the local TLB may require a change in the TLB's of other
82  * CPUs, we try to avoid sending an IPI if at all possible.  For instance, if
83  * we are updating a PTE and that PTE previously was invalid and therefore
84  * couldn't support an active mapping, there's no need for an IPI since there
85  * can't be a TLB entry to invalidate.  The other case is when we change a PTE
86  * to be modified we just update the local TLB.  If another TLB has a stale
87  * entry, a TLB MOD exception will be raised and that will cause the local TLB
88  * to be updated.
89  *
90  * We never need to update a non-local TLB if the pmap doesn't have a valid
91  * ASID for that TLB.  If it does have a valid ASID but isn't current "onproc"
92  * we simply reset its ASID for that TLB and then when it goes "onproc" it
93  * will allocate a new ASID and any existing TLB entries will be orphaned.
94  * Only in the case that pmap has an "onproc" ASID do we actually have to send
95  * an IPI.
96  *
97  * Once we determined we must send an IPI to shootdown a TLB, we need to send
98  * it to one of CPUs that share that TLB.  We choose the lowest numbered CPU
99  * that has one of the pmap's ASID "onproc".  In reality, any CPU sharing that
100  * TLB would do, but interrupting an active CPU seems best.
101  *
102  * A TLB might have multiple shootdowns active concurrently.  The shootdown
103  * logic compresses these into a few cases:
104  *	0) nobody needs to have its TLB entries invalidated
105  *	1) one ASID needs to have its TLB entries invalidated
106  *	2) more than one ASID needs to have its TLB entries invalidated
107  *	3) the kernel needs to have its TLB entries invalidated
108  *	4) the kernel and one or more ASID need their TLB entries invalidated.
109  *
110  * And for each case we do:
111  *	0) nothing,
112  *	1) if that ASID is still "onproc", we invalidate the TLB entries for
113  *	   that single ASID.  If not, just reset the pmap's ASID to invalidate
114  *	   and let it allocate a new ASID the next time it goes "onproc",
115  *	2) we reinitialize the ASID space (preserving any "onproc" ASIDs) and
116  *	   invalidate all non-wired non-global TLB entries,
117  *	3) we invalidate all of the non-wired global TLB entries,
118  *	4) we reinitialize the ASID space (again preserving any "onproc" ASIDs)
119  *	   invalidate all non-wired TLB entries.
120  *
121  * As you can see, shootdowns are not concerned with addresses, just address
122  * spaces.  Since the number of TLB entries is usually quite small, this avoids
123  * a lot of overhead for not much gain.
124  */
125 
126 #define __PMAP_PRIVATE
127 
128 #include "opt_multiprocessor.h"
129 
130 #include <sys/param.h>
131 
132 #include <sys/atomic.h>
133 #include <sys/cpu.h>
134 #include <sys/kernel.h>			/* for cold */
135 #include <sys/mutex.h>
136 #include <sys/proc.h>
137 #include <sys/systm.h>
138 
139 #include <uvm/uvm.h>
140 
141 static kmutex_t pmap_tlb0_lock __cacheline_aligned;
142 
143 #define	IFCONSTANT(x)	(__builtin_constant_p((x)) ? (x) : 0)
144 
145 #if KERNEL_PID > 31
146 #error "KERNEL_PID expected in range 0-31"
147 #endif
148 
149 #define	TLBINFO_ASID_MARK_UNUSED(ti, asid) \
150 	__BITMAP_CLR((asid), &(ti)->ti_asid_bitmap)
151 #define	TLBINFO_ASID_MARK_USED(ti, asid) \
152 	__BITMAP_SET((asid), &(ti)->ti_asid_bitmap)
153 #define	TLBINFO_ASID_INUSE_P(ti, asid) \
154 	__BITMAP_ISSET((asid), &(ti)->ti_asid_bitmap)
155 #define	TLBINFO_ASID_RESET(ti) \
156 	do {								\
157 		__BITMAP_ZERO(&ti->ti_asid_bitmap);			\
158 		for (tlb_asid_t asid = 0; asid <= KERNEL_PID; asid++) 	\
159 			TLBINFO_ASID_MARK_USED(ti, asid);	 	\
160 	} while (0)
161 #define	TLBINFO_ASID_INITIAL_FREE(asid_max) \
162 	(asid_max + 1 /* 0 */ - (1 + KERNEL_PID))
163 
164 struct pmap_tlb_info pmap_tlb0_info = {
165 	.ti_name = "tlb0",
166 	.ti_asid_hint = KERNEL_PID + 1,
167 #ifdef PMAP_TLB_NUM_PIDS
168 	.ti_asid_max = IFCONSTANT(PMAP_TLB_NUM_PIDS - 1),
169 	.ti_asids_free = IFCONSTANT(
170 		TLBINFO_ASID_INITIAL_FREE(PMAP_TLB_NUM_PIDS - 1)),
171 #endif
172 	.ti_asid_bitmap._b[0] = __BITS(0, KERNEL_PID),
173 #ifdef PMAP_TLB_WIRED_UPAGES
174 	.ti_wired = PMAP_TLB_WIRED_UPAGES,
175 #endif
176 	.ti_lock = &pmap_tlb0_lock,
177 	.ti_pais = LIST_HEAD_INITIALIZER(pmap_tlb0_info.ti_pais),
178 #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1
179 	.ti_tlbinvop = TLBINV_NOBODY,
180 #endif
181 };
182 
183 #undef IFCONSTANT
184 
185 #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1
186 struct pmap_tlb_info *pmap_tlbs[PMAP_TLB_MAX] = {
187 	[0] = &pmap_tlb0_info,
188 };
189 u_int pmap_ntlbs = 1;
190 #endif
191 
192 #ifdef MULTIPROCESSOR
193 __unused static inline bool
194 pmap_tlb_intersecting_active_p(pmap_t pm, struct pmap_tlb_info *ti)
195 {
196 #if PMAP_TLB_MAX == 1
197 	return !kcpuset_iszero(pm->pm_active);
198 #else
199 	return kcpuset_intersecting_p(pm->pm_active, ti->ti_kcpuset);
200 #endif
201 }
202 
203 static inline bool
204 pmap_tlb_intersecting_onproc_p(pmap_t pm, struct pmap_tlb_info *ti)
205 {
206 #if PMAP_TLB_MAX == 1
207 	return !kcpuset_iszero(pm->pm_onproc);
208 #else
209 	return kcpuset_intersecting_p(pm->pm_onproc, ti->ti_kcpuset);
210 #endif
211 }
212 #endif
213 
214 static void
215 pmap_tlb_pai_check(struct pmap_tlb_info *ti, bool locked_p)
216 {
217 #ifdef DIAGNOSTIC
218 	struct pmap_asid_info *pai;
219 	if (!locked_p)
220 		TLBINFO_LOCK(ti);
221 	LIST_FOREACH(pai, &ti->ti_pais, pai_link) {
222 		KASSERT(pai != NULL);
223 		KASSERT(PAI_PMAP(pai, ti) != pmap_kernel());
224 		KASSERT(pai->pai_asid > KERNEL_PID);
225 		KASSERTMSG(pai->pai_asid <= ti->ti_asid_max,
226 		    "pm %p asid %#x", PAI_PMAP(pai, ti), pai->pai_asid);
227 		KASSERTMSG(TLBINFO_ASID_INUSE_P(ti, pai->pai_asid),
228 		    "pm %p asid %u", PAI_PMAP(pai, ti), pai->pai_asid);
229 #ifdef MULTIPROCESSOR
230 		KASSERT(pmap_tlb_intersecting_active_p(PAI_PMAP(pai, ti), ti));
231 #endif
232 	}
233 	if (!locked_p)
234 		TLBINFO_UNLOCK(ti);
235 #endif
236 }
237 
238 static void
239 pmap_tlb_pai_reset(struct pmap_tlb_info *ti, struct pmap_asid_info *pai,
240 	struct pmap *pm)
241 {
242 	UVMHIST_FUNC(__func__);
243 	UVMHIST_CALLARGS(maphist, "(ti=%#jx, pai=%#jx, pm=%#jx): asid %u",
244 	    (uintptr_t)ti, (uintptr_t)pai, (uintptr_t)pm, pai->pai_asid);
245 
246 	/*
247 	 * We must have an ASID but it must not be onproc (on a processor).
248 	 */
249 	KASSERT(pai->pai_asid > KERNEL_PID);
250 	KASSERT(pai->pai_asid <= ti->ti_asid_max);
251 #if defined(MULTIPROCESSOR)
252 	KASSERT(pmap_tlb_intersecting_active_p(pm, ti));
253 	KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));
254 #endif
255 	LIST_REMOVE(pai, pai_link);
256 #ifdef DIAGNOSTIC
257 	pai->pai_link.le_prev = NULL;	/* tagged as unlinked */
258 #endif
259 	/*
260 	 * If the platform has a cheap way to flush ASIDs then free the ASID
261 	 * back into the pool.  On multiprocessor systems, we will flush the
262 	 * ASID from the TLB when it's allocated.  That way we know the flush
263 	 * was always done in the correct TLB space.  On uniprocessor systems,
264 	 * just do the flush now since we know that it has been used.  This has
265 	 * a bit less overhead.  Either way, this will mean that we will only
266 	 * need to flush all ASIDs if all ASIDs are in use and we need to
267 	 * allocate a new one.
268 	 */
269 	if (PMAP_TLB_FLUSH_ASID_ON_RESET) {
270 #ifndef MULTIPROCESSOR
271 		tlb_invalidate_asids(pai->pai_asid, pai->pai_asid);
272 #endif
273 		if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) {
274 			TLBINFO_ASID_MARK_UNUSED(ti, pai->pai_asid);
275 			ti->ti_asids_free++;
276 		}
277 	}
278 	/*
279 	 * Note that we don't mark the ASID as not in use in the TLB's ASID
280 	 * bitmap (thus it can't be allocated until the ASID space is exhausted
281 	 * and therefore reinitialized).  We don't want to flush the TLB for
282 	 * entries belonging to this ASID so we will let natural TLB entry
283 	 * replacement flush them out of the TLB.  Any new entries for this
284 	 * pmap will need a new ASID allocated.
285 	 */
286 	pai->pai_asid = 0;
287 
288 #if defined(MULTIPROCESSOR)
289 	/*
290 	 * The bits in pm_active belonging to this TLB can only be changed
291 	 * while this TLB's lock is held.
292 	 */
293 #if PMAP_TLB_MAX == 1
294 	kcpuset_zero(pm->pm_active);
295 #else
296 	kcpuset_remove(pm->pm_active, ti->ti_kcpuset);
297 #endif
298 	KASSERT(!pmap_tlb_intersecting_active_p(pm, ti));
299 #endif /* MULTIPROCESSOR */
300 
301 	UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
302 }
303 
304 void
305 pmap_tlb_info_evcnt_attach(struct pmap_tlb_info *ti)
306 {
307 #if defined(MULTIPROCESSOR) && !defined(PMAP_TLB_NO_SYNCI_EVCNT)
308 	evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_desired,
309 	    EVCNT_TYPE_MISC, NULL,
310 	    ti->ti_name, "icache syncs desired");
311 	evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_asts,
312 	    EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired,
313 	    ti->ti_name, "icache sync asts");
314 	evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_all,
315 	    EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_asts,
316 	    ti->ti_name, "icache full syncs");
317 	evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_pages,
318 	    EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_asts,
319 	    ti->ti_name, "icache pages synced");
320 	evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_duplicate,
321 	    EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired,
322 	    ti->ti_name, "icache dup pages skipped");
323 	evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_deferred,
324 	    EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired,
325 	    ti->ti_name, "icache pages deferred");
326 #endif /* MULTIPROCESSOR && !PMAP_TLB_NO_SYNCI_EVCNT */
327 	evcnt_attach_dynamic_nozero(&ti->ti_evcnt_asid_reinits,
328 	    EVCNT_TYPE_MISC, NULL,
329 	    ti->ti_name, "asid pool reinit");
330 }
331 
332 void
333 pmap_tlb_info_init(struct pmap_tlb_info *ti)
334 {
335 #if defined(MULTIPROCESSOR)
336 #if PMAP_TLB_MAX == 1
337 	KASSERT(ti == &pmap_tlb0_info);
338 #else
339 	if (ti != &pmap_tlb0_info) {
340 		KASSERT(pmap_ntlbs < PMAP_TLB_MAX);
341 
342 		KASSERT(pmap_tlbs[pmap_ntlbs] == NULL);
343 
344 		ti->ti_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SCHED);
345 		TLBINFO_ASID_RESET(ti);
346 		ti->ti_asid_hint = KERNEL_PID + 1;
347 		ti->ti_asid_max = pmap_tlbs[0]->ti_asid_max;
348 		ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max);
349 		ti->ti_tlbinvop = TLBINV_NOBODY;
350 		ti->ti_victim = NULL;
351 		kcpuset_create(&ti->ti_kcpuset, true);
352 		ti->ti_index = pmap_ntlbs++;
353 		ti->ti_wired = 0;
354 		pmap_tlbs[ti->ti_index] = ti;
355 		snprintf(ti->ti_name, sizeof(ti->ti_name), "tlb%u",
356 		    ti->ti_index);
357 		pmap_tlb_info_evcnt_attach(ti);
358 
359 		KASSERT(ti->ti_asid_max < PMAP_TLB_BITMAP_LENGTH);
360 		return;
361 	}
362 #endif
363 #endif /* MULTIPROCESSOR */
364 	KASSERT(ti == &pmap_tlb0_info);
365 	KASSERT(ti->ti_lock == &pmap_tlb0_lock);
366 	//printf("ti_lock %p ", ti->ti_lock);
367 	mutex_init(ti->ti_lock, MUTEX_DEFAULT, IPL_SCHED);
368 #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1
369 	kcpuset_create(&ti->ti_kcpuset, true);
370 	kcpuset_set(ti->ti_kcpuset, cpu_index(curcpu()));
371 #endif
372 	//printf("asid ");
373 	if (ti->ti_asid_max == 0) {
374 		ti->ti_asid_max = pmap_md_tlb_asid_max();
375 		ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max);
376 	}
377 
378 	KASSERT(ti->ti_asid_max < PMAP_TLB_BITMAP_LENGTH);
379 }
380 
381 #if defined(MULTIPROCESSOR)
382 void
383 pmap_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci)
384 {
385 	KASSERT(!CPU_IS_PRIMARY(ci));
386 	KASSERT(ci->ci_data.cpu_idlelwp != NULL);
387 	KASSERT(cold);
388 
389 	TLBINFO_LOCK(ti);
390 #if PMAP_TLB_MAX > 1
391 	kcpuset_set(ti->ti_kcpuset, cpu_index(ci));
392 	cpu_set_tlb_info(ci, ti);
393 #endif
394 
395 	/*
396 	 * Do any MD tlb info init.
397 	 */
398 	pmap_md_tlb_info_attach(ti, ci);
399 
400 	/*
401 	 * The kernel pmap uses the kcpuset_running set so it's always
402 	 * up-to-date.
403 	 */
404 	TLBINFO_UNLOCK(ti);
405 }
406 #endif /* MULTIPROCESSOR */
407 
408 #ifdef DIAGNOSTIC
409 static size_t
410 pmap_tlb_asid_count(struct pmap_tlb_info *ti)
411 {
412 	size_t count = 0;
413 	for (tlb_asid_t asid = 1; asid <= ti->ti_asid_max; asid++) {
414 		if (TLBINFO_ASID_INUSE_P(ti, asid))
415 			count++;
416 	}
417 	return count;
418 }
419 #endif
420 
421 static void
422 pmap_tlb_asid_reinitialize(struct pmap_tlb_info *ti, enum tlb_invalidate_op op)
423 {
424 	UVMHIST_FUNC(__func__);
425 	UVMHIST_CALLARGS(maphist, "(ti=%#jx, op=%ju)", (uintptr_t)ti, op, 0, 0);
426 
427 	pmap_tlb_pai_check(ti, true);
428 
429 	ti->ti_evcnt_asid_reinits.ev_count++;
430 
431 	/*
432 	 * First, clear the ASID bitmap (except for ASID 0 which belongs
433 	 * to the kernel).
434 	 */
435 	ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max);
436 	ti->ti_asid_hint = KERNEL_PID + 1;
437 	TLBINFO_ASID_RESET(ti);
438 
439 	switch (op) {
440 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN)
441 	case TLBINV_ALL:
442 		tlb_invalidate_all();
443 		break;
444 	case TLBINV_ALLUSER:
445 		tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max);
446 		break;
447 #endif /* MULTIPROCESSOR && PMAP_TLB_NEED_SHOOTDOWN */
448 	case TLBINV_NOBODY: {
449 		/*
450 		 * If we are just reclaiming ASIDs in the TLB, let's go find
451 		 * what ASIDs are in use in the TLB.  Since this is a
452 		 * semi-expensive operation, we don't want to do it too often.
453 		 * So if more half of the ASIDs are in use, we don't have
454 		 * enough free ASIDs so invalidate the TLB entries with ASIDs
455 		 * and clear the ASID bitmap.  That will force everyone to
456 		 * allocate a new ASID.
457 		 */
458 #if !defined(MULTIPROCESSOR) || defined(PMAP_TLB_NEED_SHOOTDOWN)
459 		pmap_tlb_asid_check();
460 		const u_int asids_found = tlb_record_asids(
461 		    ti->ti_asid_bitmap._b, ti->ti_asid_max);
462 		pmap_tlb_asid_check();
463 #ifdef DIAGNOSTIC
464 		const u_int asids_count = pmap_tlb_asid_count(ti);
465 #endif
466 		KASSERTMSG(asids_found == asids_count,
467 		    "found %u != count %u", asids_found, asids_count);
468 		if (__predict_false(asids_found >= ti->ti_asid_max / 2)) {
469 			tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max);
470 #else /* MULTIPROCESSOR && !PMAP_TLB_NEED_SHOOTDOWN */
471 			/*
472 			 * For those systems (PowerPC) that don't require
473 			 * cross cpu TLB shootdowns, we have to invalidate the
474 			 * entire TLB because we can't record the ASIDs in use
475 			 * on the other CPUs.  This is hopefully cheaper than
476 			 * than trying to use an IPI to record all the ASIDs
477 			 * on all the CPUs (which would be a synchronization
478 			 * nightmare).
479 			 */
480 			tlb_invalidate_all();
481 #endif /* MULTIPROCESSOR && !PMAP_TLB_NEED_SHOOTDOWN */
482 			TLBINFO_ASID_RESET(ti);
483 			ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(
484 				ti->ti_asid_max);
485 #if !defined(MULTIPROCESSOR) || defined(PMAP_TLB_NEED_SHOOTDOWN)
486 		} else {
487 			ti->ti_asids_free -= asids_found;
488 		}
489 #endif /* !MULTIPROCESSOR || PMAP_TLB_NEED_SHOOTDOWN */
490 		KASSERTMSG(ti->ti_asids_free <= ti->ti_asid_max, "%u",
491 		    ti->ti_asids_free);
492 		break;
493 	}
494 	default:
495 		panic("%s: unexpected op %d", __func__, op);
496 	}
497 
498 	/*
499 	 * Now go through the active ASIDs.  If the ASID is on a processor or
500 	 * we aren't invalidating all ASIDs and the TLB has an entry owned by
501 	 * that ASID, mark it as in use.  Otherwise release the ASID.
502 	 */
503 	struct pmap_asid_info *pai, *next;
504 	for (pai = LIST_FIRST(&ti->ti_pais); pai != NULL; pai = next) {
505 		struct pmap * const pm = PAI_PMAP(pai, ti);
506 		next = LIST_NEXT(pai, pai_link);
507 		KASSERT(pm != pmap_kernel());
508 		KASSERT(pai->pai_asid > KERNEL_PID);
509 #if defined(MULTIPROCESSOR)
510 		if (pmap_tlb_intersecting_onproc_p(pm, ti)) {
511 			if (!TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) {
512 				TLBINFO_ASID_MARK_USED(ti, pai->pai_asid);
513 				ti->ti_asids_free--;
514 			}
515 			continue;
516 		}
517 #endif /* MULTIPROCESSOR */
518 		if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) {
519 			KASSERT(op == TLBINV_NOBODY);
520 		} else {
521 			pmap_tlb_pai_reset(ti, pai, pm);
522 		}
523 	}
524 #ifdef DIAGNOSTIC
525 	size_t free_count __diagused = ti->ti_asid_max - pmap_tlb_asid_count(ti);
526 	KASSERTMSG(free_count == ti->ti_asids_free,
527 	    "bitmap error: %zu != %u", free_count, ti->ti_asids_free);
528 #endif
529 	UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
530 }
531 
532 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN)
533 #if PMAP_TLB_MAX == 1
534 #error shootdown not required for single TLB systems
535 #endif
536 void
537 pmap_tlb_shootdown_process(void)
538 {
539 	struct cpu_info * const ci = curcpu();
540 	struct pmap_tlb_info * const ti = cpu_tlb_info(ci);
541 #ifdef DIAGNOSTIC
542 	struct pmap * const pm = curlwp->l_proc->p_vmspace->vm_map.pmap;
543 #endif
544 
545 	KASSERT(cpu_intr_p());
546 	KASSERTMSG(ci->ci_cpl >= IPL_SCHED,
547 	    "%s: cpl (%d) < IPL_SCHED (%d)",
548 	    __func__, ci->ci_cpl, IPL_SCHED);
549 
550 	TLBINFO_LOCK(ti);
551 
552 	switch (ti->ti_tlbinvop) {
553 	case TLBINV_ONE: {
554 		/*
555 		 * We only need to invalidate one user ASID.
556 		 */
557 		struct pmap_asid_info * const pai = PMAP_PAI(ti->ti_victim, ti);
558 		KASSERT(ti->ti_victim != pmap_kernel());
559 		if (!pmap_tlb_intersecting_onproc_p(ti->ti_victim, ti)) {
560 			/*
561 			 * The victim is an active pmap so we will just
562 			 * invalidate its TLB entries.
563 			 */
564 			KASSERT(pai->pai_asid > KERNEL_PID);
565 			pmap_tlb_asid_check();
566 			tlb_invalidate_asids(pai->pai_asid, pai->pai_asid);
567 			pmap_tlb_asid_check();
568 		} else if (pai->pai_asid) {
569 			/*
570 			 * The victim is no longer an active pmap for this TLB.
571 			 * So simply clear its ASID and when pmap_activate is
572 			 * next called for this pmap, it will allocate a new
573 			 * ASID.
574 			 */
575 			KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));
576 			pmap_tlb_pai_reset(ti, pai, PAI_PMAP(pai, ti));
577 		}
578 		break;
579 	}
580 	case TLBINV_ALLUSER:
581 		/*
582 		 * Flush all user TLB entries.
583 		 */
584 		pmap_tlb_asid_reinitialize(ti, TLBINV_ALLUSER);
585 		break;
586 	case TLBINV_ALLKERNEL:
587 		/*
588 		 * We need to invalidate all global TLB entries.
589 		 */
590 		pmap_tlb_asid_check();
591 		tlb_invalidate_globals();
592 		pmap_tlb_asid_check();
593 		break;
594 	case TLBINV_ALL:
595 		/*
596 		 * Flush all the TLB entries (user and kernel).
597 		 */
598 		pmap_tlb_asid_reinitialize(ti, TLBINV_ALL);
599 		break;
600 	case TLBINV_NOBODY:
601 		/*
602 		 * Might be spurious or another SMT CPU sharing this TLB
603 		 * could have already done the work.
604 		 */
605 		break;
606 	}
607 
608 	/*
609 	 * Indicate we are done with shutdown event.
610 	 */
611 	ti->ti_victim = NULL;
612 	ti->ti_tlbinvop = TLBINV_NOBODY;
613 	TLBINFO_UNLOCK(ti);
614 }
615 
616 /*
617  * This state machine could be encoded into an array of integers but since all
618  * the values fit in 3 bits, the 5 entry "table" fits in a 16 bit value which
619  * can be loaded in a single instruction.
620  */
621 #define	TLBINV_MAP(op, nobody, one, alluser, allkernel, all)	\
622 	((((   (nobody) << 3*TLBINV_NOBODY)			\
623 	 | (      (one) << 3*TLBINV_ONE)			\
624 	 | (  (alluser) << 3*TLBINV_ALLUSER)			\
625 	 | ((allkernel) << 3*TLBINV_ALLKERNEL)			\
626 	 | (      (all) << 3*TLBINV_ALL)) >> 3*(op)) & 7)
627 
628 #define	TLBINV_USER_MAP(op)	\
629 	TLBINV_MAP(op, TLBINV_ONE, TLBINV_ALLUSER, TLBINV_ALLUSER,	\
630 	    TLBINV_ALL, TLBINV_ALL)
631 
632 #define	TLBINV_KERNEL_MAP(op)	\
633 	TLBINV_MAP(op, TLBINV_ALLKERNEL, TLBINV_ALL, TLBINV_ALL,	\
634 	    TLBINV_ALLKERNEL, TLBINV_ALL)
635 
636 bool
637 pmap_tlb_shootdown_bystanders(pmap_t pm)
638 {
639 	/*
640 	 * We don't need to deal with our own TLB.
641 	 */
642 
643 	UVMHIST_FUNC(__func__);
644 	UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pm, 0, 0, 0);
645 
646 	kcpuset_t *pm_active;
647 	kcpuset_clone(&pm_active, pm->pm_active);
648 	kcpuset_remove(pm_active, cpu_tlb_info(curcpu())->ti_kcpuset);
649 	const bool kernel_p = (pm == pmap_kernel());
650 	bool ipi_sent = false;
651 
652 	/*
653 	 * If pm_active gets more bits set, then it's after all our changes
654 	 * have been made so they will already be cognizant of them.
655 	 */
656 
657 	for (size_t i = 0; !kcpuset_iszero(pm_active); i++) {
658 		KASSERT(i < pmap_ntlbs);
659 		struct pmap_tlb_info * const ti = pmap_tlbs[i];
660 		KASSERT(tlbinfo_index(ti) == i);
661 		/*
662 		 * Skip this TLB if there are no active mappings for it.
663 		 */
664 		if (!kcpuset_intersecting_p(pm_active, ti->ti_kcpuset))
665 			continue;
666 		struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
667 		kcpuset_remove(pm_active, ti->ti_kcpuset);
668 		TLBINFO_LOCK(ti);
669 		cpuid_t j = kcpuset_ffs_intersecting(pm->pm_onproc,
670 		    ti->ti_kcpuset);
671 		// post decrement since ffs returns bit + 1 or 0 if no bit
672 		if (j-- > 0) {
673 			if (kernel_p) {
674 				ti->ti_tlbinvop =
675 				    TLBINV_KERNEL_MAP(ti->ti_tlbinvop);
676 				ti->ti_victim = NULL;
677 			} else {
678 				KASSERT(pai->pai_asid);
679 				if (__predict_false(ti->ti_victim == pm)) {
680 					KASSERT(ti->ti_tlbinvop == TLBINV_ONE);
681 					/*
682 					 * We still need to invalidate this one
683 					 * ASID so there's nothing to change.
684 					 */
685 				} else {
686 					ti->ti_tlbinvop =
687 					    TLBINV_USER_MAP(ti->ti_tlbinvop);
688 					if (ti->ti_tlbinvop == TLBINV_ONE)
689 						ti->ti_victim = pm;
690 					else
691 						ti->ti_victim = NULL;
692 				}
693 			}
694 			TLBINFO_UNLOCK(ti);
695 			/*
696 			 * Now we can send out the shootdown IPIs to a CPU
697 			 * that shares this TLB and is currently using this
698 			 * pmap.  That CPU will process the IPI and do the
699 			 * all the work.  Any other CPUs sharing that TLB
700 			 * will take advantage of that work.  pm_onproc might
701 			 * change now that we have released the lock but we
702 			 * can tolerate spurious shootdowns.
703 			 */
704 			cpu_send_ipi(cpu_lookup(j), IPI_SHOOTDOWN);
705 			ipi_sent = true;
706 			continue;
707 		}
708 		if (!pmap_tlb_intersecting_active_p(pm, ti)) {
709 			/*
710 			 * If this pmap has an ASID assigned but it's not
711 			 * currently running, nuke its ASID.  Next time the
712 			 * pmap is activated, it will allocate a new ASID.
713 			 * And best of all, we avoid an IPI.
714 			 */
715 			KASSERT(!kernel_p);
716 			pmap_tlb_pai_reset(ti, pai, pm);
717 			//ti->ti_evcnt_lazy_shots.ev_count++;
718 		}
719 		TLBINFO_UNLOCK(ti);
720 	}
721 
722 	kcpuset_destroy(pm_active);
723 
724 	UVMHIST_LOG(maphist, " <-- done (ipi_sent=%jd)", ipi_sent, 0, 0, 0);
725 
726 	return ipi_sent;
727 }
728 #endif /* MULTIPROCESSOR && PMAP_TLB_NEED_SHOOTDOWN */
729 
730 #ifndef PMAP_HWPAGEWALKER
731 int
732 pmap_tlb_update_addr(pmap_t pm, vaddr_t va, pt_entry_t pte, u_int flags)
733 {
734 	struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu());
735 	struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
736 	int rv = -1;
737 
738 	UVMHIST_FUNC(__func__);
739 	UVMHIST_CALLARGS(maphist, " (pm=%#jx va=%#jx, pte=%#jx flags=%#jx)",
740 	    (uintptr_t)pm, va, pte_value(pte), flags);
741 
742 	KASSERT(kpreempt_disabled());
743 
744 	KASSERTMSG(pte_valid_p(pte), "va %#"PRIxVADDR" %#"PRIxPTE,
745 	    va, pte_value(pte));
746 
747 	TLBINFO_LOCK(ti);
748 	if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) {
749 		pmap_tlb_asid_check();
750 		rv = tlb_update_addr(va, pai->pai_asid, pte,
751 		    (flags & PMAP_TLB_INSERT) != 0);
752 		pmap_tlb_asid_check();
753 		UVMHIST_LOG(maphist,
754 		     "   %jd <-- tlb_update_addr(%#jx, %#jx, %#jx, ...)",
755 		     rv, va, pai->pai_asid, pte_value(pte));
756 		KASSERTMSG((flags & PMAP_TLB_INSERT) == 0 || rv == 1,
757 		    "pmap %p (asid %u) va %#"PRIxVADDR" pte %#"PRIxPTE" rv %d",
758 		    pm, pai->pai_asid, va, pte_value(pte), rv);
759 	}
760 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN)
761 	if (flags & PMAP_TLB_NEED_IPI)
762 		pm->pm_shootdown_pending = 1;
763 #endif
764 	TLBINFO_UNLOCK(ti);
765 
766 	UVMHIST_LOG(maphist, "   <-- done (rv=%jd)", rv, 0, 0, 0);
767 
768 	return rv;
769 }
770 #endif /* !PMAP_HWPAGEWALKER */
771 
772 void
773 pmap_tlb_invalidate_addr(pmap_t pm, vaddr_t va)
774 {
775 	struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu());
776 	struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
777 
778 	UVMHIST_FUNC(__func__);
779 	UVMHIST_CALLARGS(maphist, " (pm=%#jx va=%#jx) ti=%#jx asid=%#jx",
780 	    (uintptr_t)pm, va, (uintptr_t)ti, pai->pai_asid);
781 
782 	KASSERT(kpreempt_disabled());
783 
784 	TLBINFO_LOCK(ti);
785 	if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) {
786 		pmap_tlb_asid_check();
787 		UVMHIST_LOG(maphist, " invalidating %#jx asid %#jx",
788 		    va, pai->pai_asid, 0, 0);
789 		tlb_invalidate_addr(va, pai->pai_asid);
790 		pmap_tlb_asid_check();
791 	}
792 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN)
793 	pm->pm_shootdown_pending = 1;
794 #endif
795 	TLBINFO_UNLOCK(ti);
796 	UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
797 }
798 
799 static inline void
800 pmap_tlb_asid_alloc(struct pmap_tlb_info *ti, pmap_t pm,
801 	struct pmap_asid_info *pai)
802 {
803 	/*
804 	 * We shouldn't have an ASID assigned, and thusly must not be onproc
805 	 * nor active.
806 	 */
807 	KASSERT(pm != pmap_kernel());
808 	KASSERT(pai->pai_asid == 0);
809 	KASSERT(pai->pai_link.le_prev == NULL);
810 #if defined(MULTIPROCESSOR)
811 	KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));
812 	KASSERT(!pmap_tlb_intersecting_active_p(pm, ti));
813 #endif
814 	KASSERT(ti->ti_asids_free > 0);
815 	KASSERT(ti->ti_asid_hint > KERNEL_PID);
816 
817 	/*
818 	 * If the last ASID allocated was the maximum ASID, then the
819 	 * hint will be out of range.  Reset the hint to first
820 	 * available ASID.
821 	 */
822 	if (PMAP_TLB_FLUSH_ASID_ON_RESET
823 	    && ti->ti_asid_hint > ti->ti_asid_max) {
824 		ti->ti_asid_hint = KERNEL_PID + 1;
825 	}
826 	KASSERTMSG(ti->ti_asid_hint <= ti->ti_asid_max, "hint %u",
827 	    ti->ti_asid_hint);
828 
829 	/*
830 	 * Let's see if the hinted ASID is free.  If not search for
831 	 * a new one.
832 	 */
833 	if (__predict_true(TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint))) {
834 		const size_t nbpw = NBBY * sizeof(ti->ti_asid_bitmap._b[0]);
835 		size_t i;
836 		u_long bits;
837 		for (i = 0; (bits = ~ti->ti_asid_bitmap._b[i]) == 0; i++) {
838 			KASSERT(i < __arraycount(ti->ti_asid_bitmap._b) - 1);
839 		}
840 		/*
841 		 * ffs wants to find the first bit set while we want
842 		 * to find the first bit cleared.
843 		 */
844 		const u_int n = __builtin_ffsl(bits) - 1;
845 		KASSERTMSG((bits << (nbpw - (n+1))) == (1ul << (nbpw-1)),
846 		    "n %u bits %#lx", n, bits);
847 		KASSERT(n < nbpw);
848 		ti->ti_asid_hint = n + i * nbpw;
849 	}
850 
851 	KASSERT(ti->ti_asid_hint > KERNEL_PID);
852 	KASSERT(ti->ti_asid_hint <= ti->ti_asid_max);
853 	KASSERTMSG(PMAP_TLB_FLUSH_ASID_ON_RESET
854 	    || TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint - 1),
855 	    "hint %u bitmap %p", ti->ti_asid_hint, &ti->ti_asid_bitmap);
856 	KASSERTMSG(!TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint),
857 	    "hint %u bitmap %p", ti->ti_asid_hint, &ti->ti_asid_bitmap);
858 
859 	/*
860 	 * The hint contains our next ASID so take it and advance the hint.
861 	 * Mark it as used and insert the pai into the list of active asids.
862 	 * There is also one less asid free in this TLB.
863 	 */
864 	KASSERT(ti->ti_asid_hint > KERNEL_PID);
865 	pai->pai_asid = ti->ti_asid_hint++;
866 #ifdef MULTIPROCESSOR
867 	if (PMAP_TLB_FLUSH_ASID_ON_RESET) {
868 		/*
869 		 * Clean the new ASID from the TLB.
870 		 */
871 		tlb_invalidate_asids(pai->pai_asid, pai->pai_asid);
872 	}
873 #endif
874 	TLBINFO_ASID_MARK_USED(ti, pai->pai_asid);
875 	LIST_INSERT_HEAD(&ti->ti_pais, pai, pai_link);
876 	ti->ti_asids_free--;
877 
878 #if defined(MULTIPROCESSOR)
879 	/*
880 	 * Mark that we now have an active ASID for all CPUs sharing this TLB.
881 	 * The bits in pm_active belonging to this TLB can only be changed
882 	 * while this TLBs lock is held.
883 	 */
884 #if PMAP_TLB_MAX == 1
885 	kcpuset_copy(pm->pm_active, kcpuset_running);
886 #else
887 	kcpuset_merge(pm->pm_active, ti->ti_kcpuset);
888 #endif
889 #endif
890 }
891 
892 /*
893  * Acquire a TLB address space tag (called ASID or TLBPID) and return it.
894  * ASID might have already been previously acquired.
895  */
896 void
897 pmap_tlb_asid_acquire(pmap_t pm, struct lwp *l)
898 {
899 	struct cpu_info * const ci = l->l_cpu;
900 	struct pmap_tlb_info * const ti = cpu_tlb_info(ci);
901 	struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
902 
903 	UVMHIST_FUNC(__func__);
904 	UVMHIST_CALLARGS(maphist, "(pm=%#jx, l=%#jx, ti=%#jx)", (uintptr_t)pm,
905 	    (uintptr_t)l, (uintptr_t)ti, 0);
906 
907 	KASSERT(kpreempt_disabled());
908 
909 	/*
910 	 * Kernels use a fixed ASID and thus doesn't need to acquire one.
911 	 */
912 	if (pm == pmap_kernel()) {
913 		UVMHIST_LOG(maphist, " <-- done (kernel)", 0, 0, 0, 0);
914 		return;
915 	}
916 
917 	TLBINFO_LOCK(ti);
918 	KASSERT(pai->pai_asid <= KERNEL_PID || pai->pai_link.le_prev != NULL);
919 	KASSERT(pai->pai_asid > KERNEL_PID || pai->pai_link.le_prev == NULL);
920 	pmap_tlb_pai_check(ti, true);
921 	if (__predict_false(!PMAP_PAI_ASIDVALID_P(pai, ti))) {
922 		/*
923 		 * If we've run out ASIDs, reinitialize the ASID space.
924 		 */
925 		if (__predict_false(tlbinfo_noasids_p(ti))) {
926 			KASSERT(l == curlwp);
927 			UVMHIST_LOG(maphist, " asid reinit", 0, 0, 0, 0);
928 			pmap_tlb_asid_reinitialize(ti, TLBINV_NOBODY);
929 			KASSERT(!tlbinfo_noasids_p(ti));
930 		}
931 
932 		/*
933 		 * Get an ASID.
934 		 */
935 		pmap_tlb_asid_alloc(ti, pm, pai);
936 		UVMHIST_LOG(maphist, "allocated asid %#jx", pai->pai_asid,
937 		    0, 0, 0);
938 	}
939 	pmap_tlb_pai_check(ti, true);
940 #if defined(MULTIPROCESSOR)
941 	KASSERT(kcpuset_isset(pm->pm_active, cpu_index(ci)));
942 #endif
943 
944 	if (l == curlwp) {
945 #if defined(MULTIPROCESSOR)
946 		/*
947 		 * The bits in pm_onproc belonging to this TLB can only
948 		 * be changed while this TLBs lock is held unless atomic
949 		 * operations are used.
950 		 */
951 		KASSERT(pm != pmap_kernel());
952 		kcpuset_atomic_set(pm->pm_onproc, cpu_index(ci));
953 #endif
954 		ci->ci_pmap_asid_cur = pai->pai_asid;
955 		UVMHIST_LOG(maphist, "setting asid to %#jx", pai->pai_asid,
956 		    0, 0, 0);
957 		tlb_set_asid(pai->pai_asid);
958 		pmap_tlb_asid_check();
959 	} else {
960 		printf("%s: l (%p) != curlwp %p\n", __func__, l, curlwp);
961 	}
962 	TLBINFO_UNLOCK(ti);
963 	UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
964 }
965 
966 void
967 pmap_tlb_asid_deactivate(pmap_t pm)
968 {
969 	UVMHIST_FUNC(__func__);
970 	UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pm, 0, 0, 0);
971 
972 	KASSERT(kpreempt_disabled());
973 #if defined(MULTIPROCESSOR)
974 	/*
975 	 * The kernel pmap is aways onproc and active and must never have
976 	 * those bits cleared.  If pmap_remove_all was called, it has already
977 	 * deactivated the pmap and thusly onproc will be 0 so there's nothing
978 	 * to do.
979 	 */
980 	if (pm != pmap_kernel() && !kcpuset_iszero(pm->pm_onproc)) {
981 		struct cpu_info * const ci = curcpu();
982 		KASSERT(!cpu_intr_p());
983 		KASSERTMSG(kcpuset_isset(pm->pm_onproc, cpu_index(ci)),
984 		    "%s: pmap %p onproc %p doesn't include cpu %d (%p)",
985 		    __func__, pm, pm->pm_onproc, cpu_index(ci), ci);
986 		/*
987 		 * The bits in pm_onproc that belong to this TLB can
988 		 * be changed while this TLBs lock is not held as long
989 		 * as we use atomic ops.
990 		 */
991 		kcpuset_atomic_clear(pm->pm_onproc, cpu_index(ci));
992 	}
993 #endif
994 	curcpu()->ci_pmap_asid_cur = KERNEL_PID;
995 	UVMHIST_LOG(maphist, " <-- done (pm=%#jx)", (uintptr_t)pm, 0, 0, 0);
996 	tlb_set_asid(KERNEL_PID);
997 	pmap_tlb_pai_check(cpu_tlb_info(curcpu()), false);
998 #if defined(DEBUG)
999 	pmap_tlb_asid_check();
1000 #endif
1001 }
1002 
1003 void
1004 pmap_tlb_asid_release_all(struct pmap *pm)
1005 {
1006 	UVMHIST_FUNC(__func__);
1007 	UVMHIST_CALLARGS(maphist, "(pm=%#jx)", (uintptr_t)pm, 0, 0, 0);
1008 
1009 	KASSERT(pm != pmap_kernel());
1010 #if defined(MULTIPROCESSOR)
1011 	//KASSERT(!kcpuset_iszero(pm->pm_onproc)); // XXX
1012 	struct cpu_info * const ci __diagused = curcpu();
1013 	KASSERT(!kcpuset_isotherset(pm->pm_onproc, cpu_index(ci)));
1014 #if PMAP_TLB_MAX > 1
1015 	for (u_int i = 0; !kcpuset_iszero(pm->pm_active); i++) {
1016 		KASSERT(i < pmap_ntlbs);
1017 		struct pmap_tlb_info * const ti = pmap_tlbs[i];
1018 #else
1019 		struct pmap_tlb_info * const ti = &pmap_tlb0_info;
1020 #endif
1021 		struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
1022 		TLBINFO_LOCK(ti);
1023 		if (PMAP_PAI_ASIDVALID_P(pai, ti)) {
1024 			/*
1025 			 * This pmap should not be in use by any other cpu so
1026 			 * we can just reset and be happy.
1027 			 */
1028 			if (ti->ti_victim == pm)
1029 				ti->ti_victim = NULL;
1030 			pmap_tlb_pai_reset(ti, pai, pm);
1031 		}
1032 		KASSERT(pai->pai_link.le_prev == NULL);
1033 		TLBINFO_UNLOCK(ti);
1034 #if PMAP_TLB_MAX > 1
1035 	}
1036 #endif
1037 #ifdef DIAGNOSTIC
1038 	for (size_t i = 0; i < (PMAP_TLB_MAX > 1 ? pmap_ntlbs : 1); i++) {
1039 		KASSERTMSG(pm->pm_pai[i].pai_asid == 0,
1040 		    "pm %p i %zu asid %u",
1041 		    pm, i, pm->pm_pai[i].pai_asid);
1042 	}
1043 #endif
1044 #else
1045 	/*
1046 	 * Handle the case of an UP kernel which only has, at most, one TLB.
1047 	 * If the pmap has an ASID allocated, free it.
1048 	 */
1049 	struct pmap_tlb_info * const ti = &pmap_tlb0_info;
1050 	struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
1051 	TLBINFO_LOCK(ti);
1052 	if (pai->pai_asid > KERNEL_PID) {
1053 		if (curcpu()->ci_pmap_asid_cur == pai->pai_asid) {
1054 			tlb_invalidate_asids(pai->pai_asid, pai->pai_asid);
1055 		} else {
1056 			pmap_tlb_pai_reset(ti, pai, pm);
1057 		}
1058 	}
1059 	TLBINFO_UNLOCK(ti);
1060 #endif /* MULTIPROCESSOR */
1061 	UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
1062 }
1063 
1064 void
1065 pmap_tlb_asid_check(void)
1066 {
1067 #ifdef DEBUG
1068 	kpreempt_disable();
1069 	const tlb_asid_t asid __debugused = tlb_get_asid();
1070 	KDASSERTMSG(asid == curcpu()->ci_pmap_asid_cur,
1071 	   "%s: asid (%#x) != current asid (%#x)",
1072 	    __func__, asid, curcpu()->ci_pmap_asid_cur);
1073 	kpreempt_enable();
1074 #endif
1075 }
1076 
1077 #ifdef DEBUG
1078 void
1079 pmap_tlb_check(pmap_t pm, bool (*func)(void *, vaddr_t, tlb_asid_t, pt_entry_t))
1080 {
1081         struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu());
1082         struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
1083         TLBINFO_LOCK(ti);
1084         if (pm == pmap_kernel() || pai->pai_asid > KERNEL_PID)
1085 		tlb_walk(pm, func);
1086         TLBINFO_UNLOCK(ti);
1087 }
1088 #endif /* DEBUG */
1089