194 pmap_tlb_intersecting_active_p(pmap_t pm, struct pmap_tlb_info *ti)  in pmap_tlb_intersecting_active_p()  argument
197 	return !kcpuset_iszero(pm->pm_active);  in pmap_tlb_intersecting_active_p()
199 	return kcpuset_intersecting_p(pm->pm_active, ti->ti_kcpuset);  in pmap_tlb_intersecting_active_p()
204 pmap_tlb_intersecting_onproc_p(pmap_t pm, struct pmap_tlb_info *ti)  in pmap_tlb_intersecting_onproc_p()  argument
207 	return !kcpuset_iszero(pm->pm_onproc);  in pmap_tlb_intersecting_onproc_p()
209 	return kcpuset_intersecting_p(pm->pm_onproc, ti->ti_kcpuset);  in pmap_tlb_intersecting_onproc_p()
244 	struct pmap *pm)  in pmap_tlb_pai_reset()  argument
248 	    (uintptr_t)ti, (uintptr_t)pai, (uintptr_t)pm, pai->pai_asid);  in pmap_tlb_pai_reset()
256 	KASSERT(pmap_tlb_intersecting_active_p(pm, ti));  in pmap_tlb_pai_reset()
257 	KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));  in pmap_tlb_pai_reset()
302 	kcpuset_zero(pm->pm_active);  in pmap_tlb_pai_reset()
304 	kcpuset_remove(pm->pm_active, ti->ti_kcpuset);  in pmap_tlb_pai_reset()
306 	KASSERT(!pmap_tlb_intersecting_active_p(pm, ti));  in pmap_tlb_pai_reset()
515 		struct pmap * const pm = PAI_PMAP(pai, ti);  in pmap_tlb_asid_reinitialize()  local
517 		KASSERT(pm != pmap_kernel());  in pmap_tlb_asid_reinitialize()
520 		if (pmap_tlb_intersecting_onproc_p(pm, ti)) {  in pmap_tlb_asid_reinitialize()
531 			pmap_tlb_pai_reset(ti, pai, pm);  in pmap_tlb_asid_reinitialize()
649 pmap_tlb_shootdown_bystanders(pmap_t pm)  in pmap_tlb_shootdown_bystanders()  argument
656 	UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pm, 0, 0, 0);  in pmap_tlb_shootdown_bystanders()
663 	kcpuset_copy(pm_active, pm->pm_active);  in pmap_tlb_shootdown_bystanders()
665 	const bool kernel_p = (pm == pmap_kernel());  in pmap_tlb_shootdown_bystanders()
683 		struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);  in pmap_tlb_shootdown_bystanders()
686 		cpuid_t j = kcpuset_ffs_intersecting(pm->pm_onproc,  in pmap_tlb_shootdown_bystanders()
696 				if (__predict_false(ti->ti_victim == pm)) {  in pmap_tlb_shootdown_bystanders()
706 						ti->ti_victim = pm;  in pmap_tlb_shootdown_bystanders()
727 		if (!pmap_tlb_intersecting_active_p(pm, ti)) {  in pmap_tlb_shootdown_bystanders()
728 			UVMHIST_LOG(maphist, "pm %#jx not active", (uintptr_t)pm, 0, 0, 0);  in pmap_tlb_shootdown_bystanders()
736 			pmap_tlb_pai_reset(ti, pai, pm);  in pmap_tlb_shootdown_bystanders()
749 pmap_tlb_update_addr(pmap_t pm, vaddr_t va, pt_entry_t pte, u_int flags)  in pmap_tlb_update_addr()  argument
754 	struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);  in pmap_tlb_update_addr()
759 	    (uintptr_t)pm, va, pte_value(pte), flags);  in pmap_tlb_update_addr()
765 	if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) {  in pmap_tlb_update_addr()
775 		    pm, pai->pai_asid, va, pte_value(pte), rv);  in pmap_tlb_update_addr()
779 		pm->pm_shootdown_pending = 1;  in pmap_tlb_update_addr()
789 pmap_tlb_invalidate_addr(pmap_t pm, vaddr_t va)  in pmap_tlb_invalidate_addr()  argument
794 	struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);  in pmap_tlb_invalidate_addr()
798 	    (uintptr_t)pm, va, (uintptr_t)ti, pai->pai_asid);  in pmap_tlb_invalidate_addr()
801 	if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) {  in pmap_tlb_invalidate_addr()
809 	pm->pm_shootdown_pending = 1;  in pmap_tlb_invalidate_addr()
816 pmap_tlb_asid_alloc(struct pmap_tlb_info *ti, pmap_t pm,  in pmap_tlb_asid_alloc()  argument
823 	KASSERT(pm != pmap_kernel());  in pmap_tlb_asid_alloc()
827 	KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));  in pmap_tlb_asid_alloc()
828 	KASSERT(!pmap_tlb_intersecting_active_p(pm, ti));  in pmap_tlb_asid_alloc()
900 	kcpuset_copy(pm->pm_active, kcpuset_running);  in pmap_tlb_asid_alloc()
902 	kcpuset_merge(pm->pm_active, ti->ti_kcpuset);  in pmap_tlb_asid_alloc()
912 pmap_tlb_asid_acquire(pmap_t pm, struct lwp *l)  in pmap_tlb_asid_acquire()  argument
918 	struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);  in pmap_tlb_asid_acquire()
921 	UVMHIST_CALLARGS(maphist, "(pm=%#jx, l=%#jx, ti=%#jx)", (uintptr_t)pm,  in pmap_tlb_asid_acquire()
927 	if (pm == pmap_kernel()) {  in pmap_tlb_asid_acquire()
945 		kcpuset_copy(pm->pm_active, kcpuset_running);  in pmap_tlb_asid_acquire()
947 		kcpuset_merge(pm->pm_active, ti->ti_kcpuset);  in pmap_tlb_asid_acquire()
964 		pmap_tlb_asid_alloc(ti, pm, pai);  in pmap_tlb_asid_acquire()
970 	KASSERT(kcpuset_isset(pm->pm_active, cpu_index(ci)));  in pmap_tlb_asid_acquire()
980 		KASSERT(pm != pmap_kernel());  in pmap_tlb_asid_acquire()
981 		kcpuset_atomic_set(pm->pm_onproc, cpu_index(ci));  in pmap_tlb_asid_acquire()
986 		tlb_set_asid(pai->pai_asid, pm);  in pmap_tlb_asid_acquire()
996 pmap_tlb_asid_deactivate(pmap_t pm)  in pmap_tlb_asid_deactivate()  argument
999 	UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pm, 0, 0, 0);  in pmap_tlb_asid_deactivate()
1009 	if (pm != pmap_kernel() && !kcpuset_iszero(pm->pm_onproc)) {  in pmap_tlb_asid_deactivate()
1012 		KASSERTMSG(kcpuset_isset(pm->pm_onproc, cpu_index(ci)),  in pmap_tlb_asid_deactivate()
1014 		    __func__, pm, pm->pm_onproc, cpu_index(ci), ci);  in pmap_tlb_asid_deactivate()
1020 		kcpuset_atomic_clear(pm->pm_onproc, cpu_index(ci));  in pmap_tlb_asid_deactivate()
1030 	UVMHIST_LOG(maphist, " <-- done (pm=%#jx)", (uintptr_t)pm, 0, 0, 0);  in pmap_tlb_asid_deactivate()
1034 pmap_tlb_asid_release_all(struct pmap *pm)  in pmap_tlb_asid_release_all()  argument
1037 	UVMHIST_CALLARGS(maphist, "(pm=%#jx)", (uintptr_t)pm, 0, 0, 0);  in pmap_tlb_asid_release_all()
1039 	KASSERT(pm != pmap_kernel());  in pmap_tlb_asid_release_all()
1043 	KASSERT(!kcpuset_isotherset(pm->pm_onproc, cpu_index(ci)));  in pmap_tlb_asid_release_all()
1045 	for (u_int i = 0; !kcpuset_iszero(pm->pm_active); i++) {  in pmap_tlb_asid_release_all()
1051 		struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);  in pmap_tlb_asid_release_all()
1058 			if (ti->ti_victim == pm)  in pmap_tlb_asid_release_all()
1061 				pmap_tlb_pai_reset(ti, pai, pm);  in pmap_tlb_asid_release_all()
1070 		KASSERTMSG(pm->pm_pai[i].pai_asid == 0,  in pmap_tlb_asid_release_all()
1072 		    pm, i, pm->pm_pai[i].pai_asid);  in pmap_tlb_asid_release_all()
1081 	struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);  in pmap_tlb_asid_release_all()
1087 			pmap_tlb_pai_reset(ti, pai, pm);  in pmap_tlb_asid_release_all()
1116 pmap_tlb_check(pmap_t pm, bool (*func)(void *, vaddr_t, tlb_asid_t, pt_entry_t))  in pmap_tlb_check()  argument
1119 	struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);  in pmap_tlb_check()
1121 	if (pm == pmap_kernel() || pai->pai_asid > KERNEL_PID)  in pmap_tlb_check()
1122 		tlb_walk(pm, func);  in pmap_tlb_check()
1129 pmap_db_tlb_print(struct pmap *pm,
1133 	pr(" asid %5u\n", pm->pm_pai[0].pai_asid);
1136 		pr(" tlb %zu  asid %5u\n", i, pm->pm_pai[i].pai_asid);