1 /* $NetBSD: pmap_tlb.c,v 1.33 2020/04/14 05:43:57 skrll Exp $ */ 2 3 /*- 4 * Copyright (c) 2010 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Matt Thomas at 3am Software Foundry. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 34 __KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.33 2020/04/14 05:43:57 skrll Exp $"); 35 36 /* 37 * Manages address spaces in a TLB. 38 * 39 * Normally there is a 1:1 mapping between a TLB and a CPU. However, some 40 * implementations may share a TLB between multiple CPUs (really CPU thread 41 * contexts). This requires the TLB abstraction to be separated from the 42 * CPU abstraction. It also requires that the TLB be locked while doing 43 * TLB activities. 44 * 45 * For each TLB, we track the ASIDs in use in a bitmap and a list of pmaps 46 * that have a valid ASID. 47 * 48 * We allocate ASIDs in increasing order until we have exhausted the supply, 49 * then reinitialize the ASID space, and start allocating again at 1. When 50 * allocating from the ASID bitmap, we skip any ASID who has a corresponding 51 * bit set in the ASID bitmap. Eventually this causes the ASID bitmap to fill 52 * and, when completely filled, a reinitialization of the ASID space. 53 * 54 * To reinitialize the ASID space, the ASID bitmap is reset and then the ASIDs 55 * of non-kernel TLB entries get recorded in the ASID bitmap. If the entries 56 * in TLB consume more than half of the ASID space, all ASIDs are invalidated, 57 * the ASID bitmap is recleared, and the list of pmaps is emptied. Otherwise, 58 * (the normal case), any ASID present in the TLB (even those which are no 59 * longer used by a pmap) will remain active (allocated) and all other ASIDs 60 * will be freed. If the size of the TLB is much smaller than the ASID space, 61 * this algorithm completely avoids TLB invalidation. 62 * 63 * For multiprocessors, we also have to deal TLB invalidation requests from 64 * other CPUs, some of which are dealt with the reinitialization of the ASID 65 * space. Whereas above we keep the ASIDs of those pmaps which have active 66 * TLB entries, this type of reinitialization preserves the ASIDs of any 67 * "onproc" user pmap and all other ASIDs will be freed. We must do this 68 * since we can't change the current ASID. 69 * 70 * Each pmap has two bitmaps: pm_active and pm_onproc. Each bit in pm_active 71 * indicates whether that pmap has an allocated ASID for a CPU. Each bit in 72 * pm_onproc indicates that the pmap's ASID is in use, i.e. a CPU has it in its 73 * "current ASID" field, e.g. the ASID field of the COP 0 register EntryHi for 74 * MIPS, or the ASID field of TTBR0 for AA64. The bit number used in these 75 * bitmaps comes from the CPU's cpu_index(). Even though these bitmaps contain 76 * the bits for all CPUs, the bits that correspond to the bits belonging to 77 * the CPUs sharing a TLB can only be manipulated while holding that TLB's 78 * lock. Atomic ops must be used to update them since multiple CPUs may be 79 * changing different sets of bits at same time but these sets never overlap. 80 * 81 * When a change to the local TLB may require a change in the TLB's of other 82 * CPUs, we try to avoid sending an IPI if at all possible. For instance, if 83 * we are updating a PTE and that PTE previously was invalid and therefore 84 * couldn't support an active mapping, there's no need for an IPI since there 85 * can't be a TLB entry to invalidate. The other case is when we change a PTE 86 * to be modified we just update the local TLB. If another TLB has a stale 87 * entry, a TLB MOD exception will be raised and that will cause the local TLB 88 * to be updated. 89 * 90 * We never need to update a non-local TLB if the pmap doesn't have a valid 91 * ASID for that TLB. If it does have a valid ASID but isn't current "onproc" 92 * we simply reset its ASID for that TLB and then when it goes "onproc" it 93 * will allocate a new ASID and any existing TLB entries will be orphaned. 94 * Only in the case that pmap has an "onproc" ASID do we actually have to send 95 * an IPI. 96 * 97 * Once we determined we must send an IPI to shootdown a TLB, we need to send 98 * it to one of CPUs that share that TLB. We choose the lowest numbered CPU 99 * that has one of the pmap's ASID "onproc". In reality, any CPU sharing that 100 * TLB would do, but interrupting an active CPU seems best. 101 * 102 * A TLB might have multiple shootdowns active concurrently. The shootdown 103 * logic compresses these into a few cases: 104 * 0) nobody needs to have its TLB entries invalidated 105 * 1) one ASID needs to have its TLB entries invalidated 106 * 2) more than one ASID needs to have its TLB entries invalidated 107 * 3) the kernel needs to have its TLB entries invalidated 108 * 4) the kernel and one or more ASID need their TLB entries invalidated. 109 * 110 * And for each case we do: 111 * 0) nothing, 112 * 1) if that ASID is still "onproc", we invalidate the TLB entries for 113 * that single ASID. If not, just reset the pmap's ASID to invalidate 114 * and let it allocate a new ASID the next time it goes "onproc", 115 * 2) we reinitialize the ASID space (preserving any "onproc" ASIDs) and 116 * invalidate all non-wired non-global TLB entries, 117 * 3) we invalidate all of the non-wired global TLB entries, 118 * 4) we reinitialize the ASID space (again preserving any "onproc" ASIDs) 119 * invalidate all non-wired TLB entries. 120 * 121 * As you can see, shootdowns are not concerned with addresses, just address 122 * spaces. Since the number of TLB entries is usually quite small, this avoids 123 * a lot of overhead for not much gain. 124 */ 125 126 #define __PMAP_PRIVATE 127 128 #include "opt_multiprocessor.h" 129 130 #include <sys/param.h> 131 132 #include <sys/atomic.h> 133 #include <sys/cpu.h> 134 #include <sys/kernel.h> /* for cold */ 135 #include <sys/mutex.h> 136 #include <sys/proc.h> 137 #include <sys/systm.h> 138 139 #include <uvm/uvm.h> 140 141 static kmutex_t pmap_tlb0_lock __cacheline_aligned; 142 143 #define IFCONSTANT(x) (__builtin_constant_p((x)) ? (x) : 0) 144 145 #if KERNEL_PID > 31 146 #error "KERNEL_PID expected in range 0-31" 147 #endif 148 149 #define TLBINFO_ASID_MARK_UNUSED(ti, asid) \ 150 __BITMAP_CLR((asid), &(ti)->ti_asid_bitmap) 151 #define TLBINFO_ASID_MARK_USED(ti, asid) \ 152 __BITMAP_SET((asid), &(ti)->ti_asid_bitmap) 153 #define TLBINFO_ASID_INUSE_P(ti, asid) \ 154 __BITMAP_ISSET((asid), &(ti)->ti_asid_bitmap) 155 #define TLBINFO_ASID_RESET(ti) \ 156 do { \ 157 __BITMAP_ZERO(&ti->ti_asid_bitmap); \ 158 for (tlb_asid_t asid = 0; asid <= KERNEL_PID; asid++) \ 159 TLBINFO_ASID_MARK_USED(ti, asid); \ 160 } while (0) 161 #define TLBINFO_ASID_INITIAL_FREE(asid_max) \ 162 (asid_max + 1 /* 0 */ - (1 + KERNEL_PID)) 163 164 struct pmap_tlb_info pmap_tlb0_info = { 165 .ti_name = "tlb0", 166 .ti_asid_hint = KERNEL_PID + 1, 167 #ifdef PMAP_TLB_NUM_PIDS 168 .ti_asid_max = IFCONSTANT(PMAP_TLB_NUM_PIDS - 1), 169 .ti_asids_free = IFCONSTANT( 170 TLBINFO_ASID_INITIAL_FREE(PMAP_TLB_NUM_PIDS - 1)), 171 #endif 172 .ti_asid_bitmap._b[0] = __BITS(0, KERNEL_PID), 173 #ifdef PMAP_TLB_WIRED_UPAGES 174 .ti_wired = PMAP_TLB_WIRED_UPAGES, 175 #endif 176 .ti_lock = &pmap_tlb0_lock, 177 .ti_pais = LIST_HEAD_INITIALIZER(pmap_tlb0_info.ti_pais), 178 #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1 179 .ti_tlbinvop = TLBINV_NOBODY, 180 #endif 181 }; 182 183 #undef IFCONSTANT 184 185 #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1 186 struct pmap_tlb_info *pmap_tlbs[PMAP_TLB_MAX] = { 187 [0] = &pmap_tlb0_info, 188 }; 189 u_int pmap_ntlbs = 1; 190 #endif 191 192 #ifdef MULTIPROCESSOR 193 __unused static inline bool 194 pmap_tlb_intersecting_active_p(pmap_t pm, struct pmap_tlb_info *ti) 195 { 196 #if PMAP_TLB_MAX == 1 197 return !kcpuset_iszero(pm->pm_active); 198 #else 199 return kcpuset_intersecting_p(pm->pm_active, ti->ti_kcpuset); 200 #endif 201 } 202 203 static inline bool 204 pmap_tlb_intersecting_onproc_p(pmap_t pm, struct pmap_tlb_info *ti) 205 { 206 #if PMAP_TLB_MAX == 1 207 return !kcpuset_iszero(pm->pm_onproc); 208 #else 209 return kcpuset_intersecting_p(pm->pm_onproc, ti->ti_kcpuset); 210 #endif 211 } 212 #endif 213 214 static void 215 pmap_tlb_pai_check(struct pmap_tlb_info *ti, bool locked_p) 216 { 217 #ifdef DIAGNOSTIC 218 struct pmap_asid_info *pai; 219 if (!locked_p) 220 TLBINFO_LOCK(ti); 221 LIST_FOREACH(pai, &ti->ti_pais, pai_link) { 222 KASSERT(pai != NULL); 223 KASSERT(PAI_PMAP(pai, ti) != pmap_kernel()); 224 KASSERT(pai->pai_asid > KERNEL_PID); 225 KASSERTMSG(pai->pai_asid <= ti->ti_asid_max, 226 "pm %p asid %#x", PAI_PMAP(pai, ti), pai->pai_asid); 227 KASSERTMSG(TLBINFO_ASID_INUSE_P(ti, pai->pai_asid), 228 "pm %p asid %u", PAI_PMAP(pai, ti), pai->pai_asid); 229 #ifdef MULTIPROCESSOR 230 KASSERT(pmap_tlb_intersecting_active_p(PAI_PMAP(pai, ti), ti)); 231 #endif 232 } 233 if (!locked_p) 234 TLBINFO_UNLOCK(ti); 235 #endif 236 } 237 238 static void 239 pmap_tlb_pai_reset(struct pmap_tlb_info *ti, struct pmap_asid_info *pai, 240 struct pmap *pm) 241 { 242 UVMHIST_FUNC(__func__); 243 UVMHIST_CALLARGS(maphist, "(ti=%#jx, pai=%#jx, pm=%#jx): asid %u", 244 (uintptr_t)ti, (uintptr_t)pai, (uintptr_t)pm, pai->pai_asid); 245 246 /* 247 * We must have an ASID but it must not be onproc (on a processor). 248 */ 249 KASSERT(pai->pai_asid > KERNEL_PID); 250 KASSERT(pai->pai_asid <= ti->ti_asid_max); 251 #if defined(MULTIPROCESSOR) 252 KASSERT(pmap_tlb_intersecting_active_p(pm, ti)); 253 KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti)); 254 #endif 255 LIST_REMOVE(pai, pai_link); 256 #ifdef DIAGNOSTIC 257 pai->pai_link.le_prev = NULL; /* tagged as unlinked */ 258 #endif 259 /* 260 * If the platform has a cheap way to flush ASIDs then free the ASID 261 * back into the pool. On multiprocessor systems, we will flush the 262 * ASID from the TLB when it's allocated. That way we know the flush 263 * was always done in the correct TLB space. On uniprocessor systems, 264 * just do the flush now since we know that it has been used. This has 265 * a bit less overhead. Either way, this will mean that we will only 266 * need to flush all ASIDs if all ASIDs are in use and we need to 267 * allocate a new one. 268 */ 269 if (PMAP_TLB_FLUSH_ASID_ON_RESET) { 270 #ifndef MULTIPROCESSOR 271 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid); 272 #endif 273 if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) { 274 TLBINFO_ASID_MARK_UNUSED(ti, pai->pai_asid); 275 ti->ti_asids_free++; 276 } 277 } 278 /* 279 * Note that we don't mark the ASID as not in use in the TLB's ASID 280 * bitmap (thus it can't be allocated until the ASID space is exhausted 281 * and therefore reinitialized). We don't want to flush the TLB for 282 * entries belonging to this ASID so we will let natural TLB entry 283 * replacement flush them out of the TLB. Any new entries for this 284 * pmap will need a new ASID allocated. 285 */ 286 pai->pai_asid = 0; 287 288 #if defined(MULTIPROCESSOR) 289 /* 290 * The bits in pm_active belonging to this TLB can only be changed 291 * while this TLB's lock is held. 292 */ 293 #if PMAP_TLB_MAX == 1 294 kcpuset_zero(pm->pm_active); 295 #else 296 kcpuset_remove(pm->pm_active, ti->ti_kcpuset); 297 #endif 298 KASSERT(!pmap_tlb_intersecting_active_p(pm, ti)); 299 #endif /* MULTIPROCESSOR */ 300 301 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 302 } 303 304 void 305 pmap_tlb_info_evcnt_attach(struct pmap_tlb_info *ti) 306 { 307 #if defined(MULTIPROCESSOR) && !defined(PMAP_TLB_NO_SYNCI_EVCNT) 308 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_desired, 309 EVCNT_TYPE_MISC, NULL, 310 ti->ti_name, "icache syncs desired"); 311 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_asts, 312 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired, 313 ti->ti_name, "icache sync asts"); 314 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_all, 315 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_asts, 316 ti->ti_name, "icache full syncs"); 317 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_pages, 318 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_asts, 319 ti->ti_name, "icache pages synced"); 320 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_duplicate, 321 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired, 322 ti->ti_name, "icache dup pages skipped"); 323 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_deferred, 324 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired, 325 ti->ti_name, "icache pages deferred"); 326 #endif /* MULTIPROCESSOR && !PMAP_TLB_NO_SYNCI_EVCNT */ 327 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_asid_reinits, 328 EVCNT_TYPE_MISC, NULL, 329 ti->ti_name, "asid pool reinit"); 330 } 331 332 void 333 pmap_tlb_info_init(struct pmap_tlb_info *ti) 334 { 335 #if defined(MULTIPROCESSOR) 336 #if PMAP_TLB_MAX == 1 337 KASSERT(ti == &pmap_tlb0_info); 338 #else 339 if (ti != &pmap_tlb0_info) { 340 KASSERT(pmap_ntlbs < PMAP_TLB_MAX); 341 342 KASSERT(pmap_tlbs[pmap_ntlbs] == NULL); 343 344 ti->ti_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SCHED); 345 TLBINFO_ASID_RESET(ti); 346 ti->ti_asid_hint = KERNEL_PID + 1; 347 ti->ti_asid_max = pmap_tlbs[0]->ti_asid_max; 348 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max); 349 ti->ti_tlbinvop = TLBINV_NOBODY; 350 ti->ti_victim = NULL; 351 kcpuset_create(&ti->ti_kcpuset, true); 352 ti->ti_index = pmap_ntlbs++; 353 ti->ti_wired = 0; 354 pmap_tlbs[ti->ti_index] = ti; 355 snprintf(ti->ti_name, sizeof(ti->ti_name), "tlb%u", 356 ti->ti_index); 357 pmap_tlb_info_evcnt_attach(ti); 358 359 KASSERT(ti->ti_asid_max < PMAP_TLB_BITMAP_LENGTH); 360 return; 361 } 362 #endif 363 #endif /* MULTIPROCESSOR */ 364 KASSERT(ti == &pmap_tlb0_info); 365 KASSERT(ti->ti_lock == &pmap_tlb0_lock); 366 //printf("ti_lock %p ", ti->ti_lock); 367 mutex_init(ti->ti_lock, MUTEX_DEFAULT, IPL_SCHED); 368 #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1 369 kcpuset_create(&ti->ti_kcpuset, true); 370 kcpuset_set(ti->ti_kcpuset, cpu_index(curcpu())); 371 #endif 372 //printf("asid "); 373 if (ti->ti_asid_max == 0) { 374 ti->ti_asid_max = pmap_md_tlb_asid_max(); 375 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max); 376 } 377 378 KASSERT(ti->ti_asid_max < PMAP_TLB_BITMAP_LENGTH); 379 } 380 381 #if defined(MULTIPROCESSOR) 382 void 383 pmap_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci) 384 { 385 KASSERT(!CPU_IS_PRIMARY(ci)); 386 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 387 KASSERT(cold); 388 389 TLBINFO_LOCK(ti); 390 #if PMAP_TLB_MAX > 1 391 kcpuset_set(ti->ti_kcpuset, cpu_index(ci)); 392 cpu_set_tlb_info(ci, ti); 393 #endif 394 395 /* 396 * Do any MD tlb info init. 397 */ 398 pmap_md_tlb_info_attach(ti, ci); 399 400 /* 401 * The kernel pmap uses the kcpuset_running set so it's always 402 * up-to-date. 403 */ 404 TLBINFO_UNLOCK(ti); 405 } 406 #endif /* MULTIPROCESSOR */ 407 408 #ifdef DIAGNOSTIC 409 static size_t 410 pmap_tlb_asid_count(struct pmap_tlb_info *ti) 411 { 412 size_t count = 0; 413 for (tlb_asid_t asid = 1; asid <= ti->ti_asid_max; asid++) { 414 if (TLBINFO_ASID_INUSE_P(ti, asid)) 415 count++; 416 } 417 return count; 418 } 419 #endif 420 421 static void 422 pmap_tlb_asid_reinitialize(struct pmap_tlb_info *ti, enum tlb_invalidate_op op) 423 { 424 UVMHIST_FUNC(__func__); 425 UVMHIST_CALLARGS(maphist, "(ti=%#jx, op=%ju)", (uintptr_t)ti, op, 0, 0); 426 427 pmap_tlb_pai_check(ti, true); 428 429 ti->ti_evcnt_asid_reinits.ev_count++; 430 431 /* 432 * First, clear the ASID bitmap (except for ASID 0 which belongs 433 * to the kernel). 434 */ 435 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max); 436 ti->ti_asid_hint = KERNEL_PID + 1; 437 TLBINFO_ASID_RESET(ti); 438 439 switch (op) { 440 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) 441 case TLBINV_ALL: 442 tlb_invalidate_all(); 443 break; 444 case TLBINV_ALLUSER: 445 tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max); 446 break; 447 #endif /* MULTIPROCESSOR && PMAP_TLB_NEED_SHOOTDOWN */ 448 case TLBINV_NOBODY: { 449 /* 450 * If we are just reclaiming ASIDs in the TLB, let's go find 451 * what ASIDs are in use in the TLB. Since this is a 452 * semi-expensive operation, we don't want to do it too often. 453 * So if more half of the ASIDs are in use, we don't have 454 * enough free ASIDs so invalidate the TLB entries with ASIDs 455 * and clear the ASID bitmap. That will force everyone to 456 * allocate a new ASID. 457 */ 458 #if !defined(MULTIPROCESSOR) || defined(PMAP_TLB_NEED_SHOOTDOWN) 459 pmap_tlb_asid_check(); 460 const u_int asids_found = tlb_record_asids( 461 ti->ti_asid_bitmap._b, ti->ti_asid_max); 462 pmap_tlb_asid_check(); 463 #ifdef DIAGNOSTIC 464 const u_int asids_count = pmap_tlb_asid_count(ti); 465 #endif 466 KASSERTMSG(asids_found == asids_count, 467 "found %u != count %u", asids_found, asids_count); 468 if (__predict_false(asids_found >= ti->ti_asid_max / 2)) { 469 tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max); 470 #else /* MULTIPROCESSOR && !PMAP_TLB_NEED_SHOOTDOWN */ 471 /* 472 * For those systems (PowerPC) that don't require 473 * cross cpu TLB shootdowns, we have to invalidate the 474 * entire TLB because we can't record the ASIDs in use 475 * on the other CPUs. This is hopefully cheaper than 476 * than trying to use an IPI to record all the ASIDs 477 * on all the CPUs (which would be a synchronization 478 * nightmare). 479 */ 480 tlb_invalidate_all(); 481 #endif /* MULTIPROCESSOR && !PMAP_TLB_NEED_SHOOTDOWN */ 482 TLBINFO_ASID_RESET(ti); 483 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE( 484 ti->ti_asid_max); 485 #if !defined(MULTIPROCESSOR) || defined(PMAP_TLB_NEED_SHOOTDOWN) 486 } else { 487 ti->ti_asids_free -= asids_found; 488 } 489 #endif /* !MULTIPROCESSOR || PMAP_TLB_NEED_SHOOTDOWN */ 490 KASSERTMSG(ti->ti_asids_free <= ti->ti_asid_max, "%u", 491 ti->ti_asids_free); 492 break; 493 } 494 default: 495 panic("%s: unexpected op %d", __func__, op); 496 } 497 498 /* 499 * Now go through the active ASIDs. If the ASID is on a processor or 500 * we aren't invalidating all ASIDs and the TLB has an entry owned by 501 * that ASID, mark it as in use. Otherwise release the ASID. 502 */ 503 struct pmap_asid_info *pai, *next; 504 for (pai = LIST_FIRST(&ti->ti_pais); pai != NULL; pai = next) { 505 struct pmap * const pm = PAI_PMAP(pai, ti); 506 next = LIST_NEXT(pai, pai_link); 507 KASSERT(pm != pmap_kernel()); 508 KASSERT(pai->pai_asid > KERNEL_PID); 509 #if defined(MULTIPROCESSOR) 510 if (pmap_tlb_intersecting_onproc_p(pm, ti)) { 511 if (!TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) { 512 TLBINFO_ASID_MARK_USED(ti, pai->pai_asid); 513 ti->ti_asids_free--; 514 } 515 continue; 516 } 517 #endif /* MULTIPROCESSOR */ 518 if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) { 519 KASSERT(op == TLBINV_NOBODY); 520 } else { 521 pmap_tlb_pai_reset(ti, pai, pm); 522 } 523 } 524 #ifdef DIAGNOSTIC 525 size_t free_count __diagused = ti->ti_asid_max - pmap_tlb_asid_count(ti); 526 KASSERTMSG(free_count == ti->ti_asids_free, 527 "bitmap error: %zu != %u", free_count, ti->ti_asids_free); 528 #endif 529 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 530 } 531 532 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) 533 #if PMAP_TLB_MAX == 1 534 #error shootdown not required for single TLB systems 535 #endif 536 void 537 pmap_tlb_shootdown_process(void) 538 { 539 struct cpu_info * const ci = curcpu(); 540 struct pmap_tlb_info * const ti = cpu_tlb_info(ci); 541 #ifdef DIAGNOSTIC 542 struct pmap * const pm = curlwp->l_proc->p_vmspace->vm_map.pmap; 543 #endif 544 545 KASSERT(cpu_intr_p()); 546 KASSERTMSG(ci->ci_cpl >= IPL_SCHED, 547 "%s: cpl (%d) < IPL_SCHED (%d)", 548 __func__, ci->ci_cpl, IPL_SCHED); 549 550 TLBINFO_LOCK(ti); 551 552 switch (ti->ti_tlbinvop) { 553 case TLBINV_ONE: { 554 /* 555 * We only need to invalidate one user ASID. 556 */ 557 struct pmap_asid_info * const pai = PMAP_PAI(ti->ti_victim, ti); 558 KASSERT(ti->ti_victim != pmap_kernel()); 559 if (!pmap_tlb_intersecting_onproc_p(ti->ti_victim, ti)) { 560 /* 561 * The victim is an active pmap so we will just 562 * invalidate its TLB entries. 563 */ 564 KASSERT(pai->pai_asid > KERNEL_PID); 565 pmap_tlb_asid_check(); 566 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid); 567 pmap_tlb_asid_check(); 568 } else if (pai->pai_asid) { 569 /* 570 * The victim is no longer an active pmap for this TLB. 571 * So simply clear its ASID and when pmap_activate is 572 * next called for this pmap, it will allocate a new 573 * ASID. 574 */ 575 KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti)); 576 pmap_tlb_pai_reset(ti, pai, PAI_PMAP(pai, ti)); 577 } 578 break; 579 } 580 case TLBINV_ALLUSER: 581 /* 582 * Flush all user TLB entries. 583 */ 584 pmap_tlb_asid_reinitialize(ti, TLBINV_ALLUSER); 585 break; 586 case TLBINV_ALLKERNEL: 587 /* 588 * We need to invalidate all global TLB entries. 589 */ 590 pmap_tlb_asid_check(); 591 tlb_invalidate_globals(); 592 pmap_tlb_asid_check(); 593 break; 594 case TLBINV_ALL: 595 /* 596 * Flush all the TLB entries (user and kernel). 597 */ 598 pmap_tlb_asid_reinitialize(ti, TLBINV_ALL); 599 break; 600 case TLBINV_NOBODY: 601 /* 602 * Might be spurious or another SMT CPU sharing this TLB 603 * could have already done the work. 604 */ 605 break; 606 } 607 608 /* 609 * Indicate we are done with shutdown event. 610 */ 611 ti->ti_victim = NULL; 612 ti->ti_tlbinvop = TLBINV_NOBODY; 613 TLBINFO_UNLOCK(ti); 614 } 615 616 /* 617 * This state machine could be encoded into an array of integers but since all 618 * the values fit in 3 bits, the 5 entry "table" fits in a 16 bit value which 619 * can be loaded in a single instruction. 620 */ 621 #define TLBINV_MAP(op, nobody, one, alluser, allkernel, all) \ 622 (((( (nobody) << 3*TLBINV_NOBODY) \ 623 | ( (one) << 3*TLBINV_ONE) \ 624 | ( (alluser) << 3*TLBINV_ALLUSER) \ 625 | ((allkernel) << 3*TLBINV_ALLKERNEL) \ 626 | ( (all) << 3*TLBINV_ALL)) >> 3*(op)) & 7) 627 628 #define TLBINV_USER_MAP(op) \ 629 TLBINV_MAP(op, TLBINV_ONE, TLBINV_ALLUSER, TLBINV_ALLUSER, \ 630 TLBINV_ALL, TLBINV_ALL) 631 632 #define TLBINV_KERNEL_MAP(op) \ 633 TLBINV_MAP(op, TLBINV_ALLKERNEL, TLBINV_ALL, TLBINV_ALL, \ 634 TLBINV_ALLKERNEL, TLBINV_ALL) 635 636 bool 637 pmap_tlb_shootdown_bystanders(pmap_t pm) 638 { 639 /* 640 * We don't need to deal with our own TLB. 641 */ 642 643 UVMHIST_FUNC(__func__); 644 UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pm, 0, 0, 0); 645 646 kcpuset_t *pm_active; 647 kcpuset_clone(&pm_active, pm->pm_active); 648 kcpuset_remove(pm_active, cpu_tlb_info(curcpu())->ti_kcpuset); 649 const bool kernel_p = (pm == pmap_kernel()); 650 bool ipi_sent = false; 651 652 /* 653 * If pm_active gets more bits set, then it's after all our changes 654 * have been made so they will already be cognizant of them. 655 */ 656 657 for (size_t i = 0; !kcpuset_iszero(pm_active); i++) { 658 KASSERT(i < pmap_ntlbs); 659 struct pmap_tlb_info * const ti = pmap_tlbs[i]; 660 KASSERT(tlbinfo_index(ti) == i); 661 /* 662 * Skip this TLB if there are no active mappings for it. 663 */ 664 if (!kcpuset_intersecting_p(pm_active, ti->ti_kcpuset)) 665 continue; 666 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 667 kcpuset_remove(pm_active, ti->ti_kcpuset); 668 TLBINFO_LOCK(ti); 669 cpuid_t j = kcpuset_ffs_intersecting(pm->pm_onproc, 670 ti->ti_kcpuset); 671 // post decrement since ffs returns bit + 1 or 0 if no bit 672 if (j-- > 0) { 673 if (kernel_p) { 674 ti->ti_tlbinvop = 675 TLBINV_KERNEL_MAP(ti->ti_tlbinvop); 676 ti->ti_victim = NULL; 677 } else { 678 KASSERT(pai->pai_asid); 679 if (__predict_false(ti->ti_victim == pm)) { 680 KASSERT(ti->ti_tlbinvop == TLBINV_ONE); 681 /* 682 * We still need to invalidate this one 683 * ASID so there's nothing to change. 684 */ 685 } else { 686 ti->ti_tlbinvop = 687 TLBINV_USER_MAP(ti->ti_tlbinvop); 688 if (ti->ti_tlbinvop == TLBINV_ONE) 689 ti->ti_victim = pm; 690 else 691 ti->ti_victim = NULL; 692 } 693 } 694 TLBINFO_UNLOCK(ti); 695 /* 696 * Now we can send out the shootdown IPIs to a CPU 697 * that shares this TLB and is currently using this 698 * pmap. That CPU will process the IPI and do the 699 * all the work. Any other CPUs sharing that TLB 700 * will take advantage of that work. pm_onproc might 701 * change now that we have released the lock but we 702 * can tolerate spurious shootdowns. 703 */ 704 cpu_send_ipi(cpu_lookup(j), IPI_SHOOTDOWN); 705 ipi_sent = true; 706 continue; 707 } 708 if (!pmap_tlb_intersecting_active_p(pm, ti)) { 709 /* 710 * If this pmap has an ASID assigned but it's not 711 * currently running, nuke its ASID. Next time the 712 * pmap is activated, it will allocate a new ASID. 713 * And best of all, we avoid an IPI. 714 */ 715 KASSERT(!kernel_p); 716 pmap_tlb_pai_reset(ti, pai, pm); 717 //ti->ti_evcnt_lazy_shots.ev_count++; 718 } 719 TLBINFO_UNLOCK(ti); 720 } 721 722 kcpuset_destroy(pm_active); 723 724 UVMHIST_LOG(maphist, " <-- done (ipi_sent=%jd)", ipi_sent, 0, 0, 0); 725 726 return ipi_sent; 727 } 728 #endif /* MULTIPROCESSOR && PMAP_TLB_NEED_SHOOTDOWN */ 729 730 #ifndef PMAP_HWPAGEWALKER 731 int 732 pmap_tlb_update_addr(pmap_t pm, vaddr_t va, pt_entry_t pte, u_int flags) 733 { 734 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 735 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 736 int rv = -1; 737 738 UVMHIST_FUNC(__func__); 739 UVMHIST_CALLARGS(maphist, " (pm=%#jx va=%#jx, pte=%#jx flags=%#jx)", 740 (uintptr_t)pm, va, pte_value(pte), flags); 741 742 KASSERT(kpreempt_disabled()); 743 744 KASSERTMSG(pte_valid_p(pte), "va %#"PRIxVADDR" %#"PRIxPTE, 745 va, pte_value(pte)); 746 747 TLBINFO_LOCK(ti); 748 if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) { 749 pmap_tlb_asid_check(); 750 rv = tlb_update_addr(va, pai->pai_asid, pte, 751 (flags & PMAP_TLB_INSERT) != 0); 752 pmap_tlb_asid_check(); 753 UVMHIST_LOG(maphist, 754 " %jd <-- tlb_update_addr(%#jx, %#jx, %#jx, ...)", 755 rv, va, pai->pai_asid, pte_value(pte)); 756 KASSERTMSG((flags & PMAP_TLB_INSERT) == 0 || rv == 1, 757 "pmap %p (asid %u) va %#"PRIxVADDR" pte %#"PRIxPTE" rv %d", 758 pm, pai->pai_asid, va, pte_value(pte), rv); 759 } 760 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) 761 if (flags & PMAP_TLB_NEED_IPI) 762 pm->pm_shootdown_pending = 1; 763 #endif 764 TLBINFO_UNLOCK(ti); 765 766 UVMHIST_LOG(maphist, " <-- done (rv=%jd)", rv, 0, 0, 0); 767 768 return rv; 769 } 770 #endif /* !PMAP_HWPAGEWALKER */ 771 772 void 773 pmap_tlb_invalidate_addr(pmap_t pm, vaddr_t va) 774 { 775 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 776 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 777 778 UVMHIST_FUNC(__func__); 779 UVMHIST_CALLARGS(maphist, " (pm=%#jx va=%#jx) ti=%#jx asid=%#jx", 780 (uintptr_t)pm, va, (uintptr_t)ti, pai->pai_asid); 781 782 KASSERT(kpreempt_disabled()); 783 784 TLBINFO_LOCK(ti); 785 if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) { 786 pmap_tlb_asid_check(); 787 UVMHIST_LOG(maphist, " invalidating %#jx asid %#jx", 788 va, pai->pai_asid, 0, 0); 789 tlb_invalidate_addr(va, pai->pai_asid); 790 pmap_tlb_asid_check(); 791 } 792 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) 793 pm->pm_shootdown_pending = 1; 794 #endif 795 TLBINFO_UNLOCK(ti); 796 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 797 } 798 799 static inline void 800 pmap_tlb_asid_alloc(struct pmap_tlb_info *ti, pmap_t pm, 801 struct pmap_asid_info *pai) 802 { 803 /* 804 * We shouldn't have an ASID assigned, and thusly must not be onproc 805 * nor active. 806 */ 807 KASSERT(pm != pmap_kernel()); 808 KASSERT(pai->pai_asid == 0); 809 KASSERT(pai->pai_link.le_prev == NULL); 810 #if defined(MULTIPROCESSOR) 811 KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti)); 812 KASSERT(!pmap_tlb_intersecting_active_p(pm, ti)); 813 #endif 814 KASSERT(ti->ti_asids_free > 0); 815 KASSERT(ti->ti_asid_hint > KERNEL_PID); 816 817 /* 818 * If the last ASID allocated was the maximum ASID, then the 819 * hint will be out of range. Reset the hint to first 820 * available ASID. 821 */ 822 if (PMAP_TLB_FLUSH_ASID_ON_RESET 823 && ti->ti_asid_hint > ti->ti_asid_max) { 824 ti->ti_asid_hint = KERNEL_PID + 1; 825 } 826 KASSERTMSG(ti->ti_asid_hint <= ti->ti_asid_max, "hint %u", 827 ti->ti_asid_hint); 828 829 /* 830 * Let's see if the hinted ASID is free. If not search for 831 * a new one. 832 */ 833 if (__predict_true(TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint))) { 834 const size_t nbpw = NBBY * sizeof(ti->ti_asid_bitmap._b[0]); 835 size_t i; 836 u_long bits; 837 for (i = 0; (bits = ~ti->ti_asid_bitmap._b[i]) == 0; i++) { 838 KASSERT(i < __arraycount(ti->ti_asid_bitmap._b) - 1); 839 } 840 /* 841 * ffs wants to find the first bit set while we want 842 * to find the first bit cleared. 843 */ 844 const u_int n = __builtin_ffsl(bits) - 1; 845 KASSERTMSG((bits << (nbpw - (n+1))) == (1ul << (nbpw-1)), 846 "n %u bits %#lx", n, bits); 847 KASSERT(n < nbpw); 848 ti->ti_asid_hint = n + i * nbpw; 849 } 850 851 KASSERT(ti->ti_asid_hint > KERNEL_PID); 852 KASSERT(ti->ti_asid_hint <= ti->ti_asid_max); 853 KASSERTMSG(PMAP_TLB_FLUSH_ASID_ON_RESET 854 || TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint - 1), 855 "hint %u bitmap %p", ti->ti_asid_hint, &ti->ti_asid_bitmap); 856 KASSERTMSG(!TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint), 857 "hint %u bitmap %p", ti->ti_asid_hint, &ti->ti_asid_bitmap); 858 859 /* 860 * The hint contains our next ASID so take it and advance the hint. 861 * Mark it as used and insert the pai into the list of active asids. 862 * There is also one less asid free in this TLB. 863 */ 864 KASSERT(ti->ti_asid_hint > KERNEL_PID); 865 pai->pai_asid = ti->ti_asid_hint++; 866 #ifdef MULTIPROCESSOR 867 if (PMAP_TLB_FLUSH_ASID_ON_RESET) { 868 /* 869 * Clean the new ASID from the TLB. 870 */ 871 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid); 872 } 873 #endif 874 TLBINFO_ASID_MARK_USED(ti, pai->pai_asid); 875 LIST_INSERT_HEAD(&ti->ti_pais, pai, pai_link); 876 ti->ti_asids_free--; 877 878 #if defined(MULTIPROCESSOR) 879 /* 880 * Mark that we now have an active ASID for all CPUs sharing this TLB. 881 * The bits in pm_active belonging to this TLB can only be changed 882 * while this TLBs lock is held. 883 */ 884 #if PMAP_TLB_MAX == 1 885 kcpuset_copy(pm->pm_active, kcpuset_running); 886 #else 887 kcpuset_merge(pm->pm_active, ti->ti_kcpuset); 888 #endif 889 #endif 890 } 891 892 /* 893 * Acquire a TLB address space tag (called ASID or TLBPID) and return it. 894 * ASID might have already been previously acquired. 895 */ 896 void 897 pmap_tlb_asid_acquire(pmap_t pm, struct lwp *l) 898 { 899 struct cpu_info * const ci = l->l_cpu; 900 struct pmap_tlb_info * const ti = cpu_tlb_info(ci); 901 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 902 903 UVMHIST_FUNC(__func__); 904 UVMHIST_CALLARGS(maphist, "(pm=%#jx, l=%#jx, ti=%#jx)", (uintptr_t)pm, 905 (uintptr_t)l, (uintptr_t)ti, 0); 906 907 KASSERT(kpreempt_disabled()); 908 909 /* 910 * Kernels use a fixed ASID and thus doesn't need to acquire one. 911 */ 912 if (pm == pmap_kernel()) { 913 UVMHIST_LOG(maphist, " <-- done (kernel)", 0, 0, 0, 0); 914 return; 915 } 916 917 TLBINFO_LOCK(ti); 918 KASSERT(pai->pai_asid <= KERNEL_PID || pai->pai_link.le_prev != NULL); 919 KASSERT(pai->pai_asid > KERNEL_PID || pai->pai_link.le_prev == NULL); 920 pmap_tlb_pai_check(ti, true); 921 if (__predict_false(!PMAP_PAI_ASIDVALID_P(pai, ti))) { 922 /* 923 * If we've run out ASIDs, reinitialize the ASID space. 924 */ 925 if (__predict_false(tlbinfo_noasids_p(ti))) { 926 KASSERT(l == curlwp); 927 UVMHIST_LOG(maphist, " asid reinit", 0, 0, 0, 0); 928 pmap_tlb_asid_reinitialize(ti, TLBINV_NOBODY); 929 KASSERT(!tlbinfo_noasids_p(ti)); 930 } 931 932 /* 933 * Get an ASID. 934 */ 935 pmap_tlb_asid_alloc(ti, pm, pai); 936 UVMHIST_LOG(maphist, "allocated asid %#jx", pai->pai_asid, 937 0, 0, 0); 938 } 939 pmap_tlb_pai_check(ti, true); 940 #if defined(MULTIPROCESSOR) 941 KASSERT(kcpuset_isset(pm->pm_active, cpu_index(ci))); 942 #endif 943 944 if (l == curlwp) { 945 #if defined(MULTIPROCESSOR) 946 /* 947 * The bits in pm_onproc belonging to this TLB can only 948 * be changed while this TLBs lock is held unless atomic 949 * operations are used. 950 */ 951 KASSERT(pm != pmap_kernel()); 952 kcpuset_atomic_set(pm->pm_onproc, cpu_index(ci)); 953 #endif 954 ci->ci_pmap_asid_cur = pai->pai_asid; 955 UVMHIST_LOG(maphist, "setting asid to %#jx", pai->pai_asid, 956 0, 0, 0); 957 tlb_set_asid(pai->pai_asid); 958 pmap_tlb_asid_check(); 959 } else { 960 printf("%s: l (%p) != curlwp %p\n", __func__, l, curlwp); 961 } 962 TLBINFO_UNLOCK(ti); 963 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 964 } 965 966 void 967 pmap_tlb_asid_deactivate(pmap_t pm) 968 { 969 UVMHIST_FUNC(__func__); 970 UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pm, 0, 0, 0); 971 972 KASSERT(kpreempt_disabled()); 973 #if defined(MULTIPROCESSOR) 974 /* 975 * The kernel pmap is aways onproc and active and must never have 976 * those bits cleared. If pmap_remove_all was called, it has already 977 * deactivated the pmap and thusly onproc will be 0 so there's nothing 978 * to do. 979 */ 980 if (pm != pmap_kernel() && !kcpuset_iszero(pm->pm_onproc)) { 981 struct cpu_info * const ci = curcpu(); 982 KASSERT(!cpu_intr_p()); 983 KASSERTMSG(kcpuset_isset(pm->pm_onproc, cpu_index(ci)), 984 "%s: pmap %p onproc %p doesn't include cpu %d (%p)", 985 __func__, pm, pm->pm_onproc, cpu_index(ci), ci); 986 /* 987 * The bits in pm_onproc that belong to this TLB can 988 * be changed while this TLBs lock is not held as long 989 * as we use atomic ops. 990 */ 991 kcpuset_atomic_clear(pm->pm_onproc, cpu_index(ci)); 992 } 993 #endif 994 curcpu()->ci_pmap_asid_cur = KERNEL_PID; 995 UVMHIST_LOG(maphist, " <-- done (pm=%#jx)", (uintptr_t)pm, 0, 0, 0); 996 tlb_set_asid(KERNEL_PID); 997 pmap_tlb_pai_check(cpu_tlb_info(curcpu()), false); 998 #if defined(DEBUG) 999 pmap_tlb_asid_check(); 1000 #endif 1001 } 1002 1003 void 1004 pmap_tlb_asid_release_all(struct pmap *pm) 1005 { 1006 UVMHIST_FUNC(__func__); 1007 UVMHIST_CALLARGS(maphist, "(pm=%#jx)", (uintptr_t)pm, 0, 0, 0); 1008 1009 KASSERT(pm != pmap_kernel()); 1010 #if defined(MULTIPROCESSOR) 1011 //KASSERT(!kcpuset_iszero(pm->pm_onproc)); // XXX 1012 struct cpu_info * const ci __diagused = curcpu(); 1013 KASSERT(!kcpuset_isotherset(pm->pm_onproc, cpu_index(ci))); 1014 #if PMAP_TLB_MAX > 1 1015 for (u_int i = 0; !kcpuset_iszero(pm->pm_active); i++) { 1016 KASSERT(i < pmap_ntlbs); 1017 struct pmap_tlb_info * const ti = pmap_tlbs[i]; 1018 #else 1019 struct pmap_tlb_info * const ti = &pmap_tlb0_info; 1020 #endif 1021 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 1022 TLBINFO_LOCK(ti); 1023 if (PMAP_PAI_ASIDVALID_P(pai, ti)) { 1024 /* 1025 * This pmap should not be in use by any other cpu so 1026 * we can just reset and be happy. 1027 */ 1028 if (ti->ti_victim == pm) 1029 ti->ti_victim = NULL; 1030 pmap_tlb_pai_reset(ti, pai, pm); 1031 } 1032 KASSERT(pai->pai_link.le_prev == NULL); 1033 TLBINFO_UNLOCK(ti); 1034 #if PMAP_TLB_MAX > 1 1035 } 1036 #endif 1037 #ifdef DIAGNOSTIC 1038 for (size_t i = 0; i < (PMAP_TLB_MAX > 1 ? pmap_ntlbs : 1); i++) { 1039 KASSERTMSG(pm->pm_pai[i].pai_asid == 0, 1040 "pm %p i %zu asid %u", 1041 pm, i, pm->pm_pai[i].pai_asid); 1042 } 1043 #endif 1044 #else 1045 /* 1046 * Handle the case of an UP kernel which only has, at most, one TLB. 1047 * If the pmap has an ASID allocated, free it. 1048 */ 1049 struct pmap_tlb_info * const ti = &pmap_tlb0_info; 1050 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 1051 TLBINFO_LOCK(ti); 1052 if (pai->pai_asid > KERNEL_PID) { 1053 if (curcpu()->ci_pmap_asid_cur == pai->pai_asid) { 1054 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid); 1055 } else { 1056 pmap_tlb_pai_reset(ti, pai, pm); 1057 } 1058 } 1059 TLBINFO_UNLOCK(ti); 1060 #endif /* MULTIPROCESSOR */ 1061 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 1062 } 1063 1064 void 1065 pmap_tlb_asid_check(void) 1066 { 1067 #ifdef DEBUG 1068 kpreempt_disable(); 1069 const tlb_asid_t asid __debugused = tlb_get_asid(); 1070 KDASSERTMSG(asid == curcpu()->ci_pmap_asid_cur, 1071 "%s: asid (%#x) != current asid (%#x)", 1072 __func__, asid, curcpu()->ci_pmap_asid_cur); 1073 kpreempt_enable(); 1074 #endif 1075 } 1076 1077 #ifdef DEBUG 1078 void 1079 pmap_tlb_check(pmap_t pm, bool (*func)(void *, vaddr_t, tlb_asid_t, pt_entry_t)) 1080 { 1081 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 1082 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 1083 TLBINFO_LOCK(ti); 1084 if (pm == pmap_kernel() || pai->pai_asid > KERNEL_PID) 1085 tlb_walk(pm, func); 1086 TLBINFO_UNLOCK(ti); 1087 } 1088 #endif /* DEBUG */ 1089