1 /* $NetBSD: pmap_tlb.c,v 1.3 2013/07/17 23:15:20 matt Exp $ */ 2 3 /*- 4 * Copyright (c) 2010 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Matt Thomas at 3am Software Foundry. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 34 __KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.3 2013/07/17 23:15:20 matt Exp $"); 35 36 /* 37 * Manages address spaces in a TLB. 38 * 39 * Normally there is a 1:1 mapping between a TLB and a CPU. However, some 40 * implementations may share a TLB between multiple CPUs (really CPU thread 41 * contexts). This requires the TLB abstraction to be separated from the 42 * CPU abstraction. It also requires that the TLB be locked while doing 43 * TLB activities. 44 * 45 * For each TLB, we track the ASIDs in use in a bitmap and a list of pmaps 46 * that have a valid ASID. 47 * 48 * We allocate ASIDs in increasing order until we have exhausted the supply, 49 * then reinitialize the ASID space, and start allocating again at 1. When 50 * allocating from the ASID bitmap, we skip any ASID who has a corresponding 51 * bit set in the ASID bitmap. Eventually this causes the ASID bitmap to fill 52 * and, when completely filled, a reinitialization of the ASID space. 53 * 54 * To reinitialize the ASID space, the ASID bitmap is reset and then the ASIDs 55 * of non-kernel TLB entries get recorded in the ASID bitmap. If the entries 56 * in TLB consume more than half of the ASID space, all ASIDs are invalidated, 57 * the ASID bitmap is recleared, and the list of pmaps is emptied. Otherwise, 58 * (the normal case), any ASID present in the TLB (even those which are no 59 * longer used by a pmap) will remain active (allocated) and all other ASIDs 60 * will be freed. If the size of the TLB is much smaller than the ASID space, 61 * this algorithm completely avoids TLB invalidation. 62 * 63 * For multiprocessors, we also have to deal TLB invalidation requests from 64 * other CPUs, some of which are dealt with the reinitialization of the ASID 65 * space. Whereas above we keep the ASIDs of those pmaps which have active 66 * TLB entries, this type of reinitialization preserves the ASIDs of any 67 * "onproc" user pmap and all other ASIDs will be freed. We must do this 68 * since we can't change the current ASID. 69 * 70 * Each pmap has two bitmaps: pm_active and pm_onproc. Each bit in pm_active 71 * indicates whether that pmap has an allocated ASID for a CPU. Each bit in 72 * pm_onproc indicates that pmap's ASID is active (equal to the ASID in COP 0 73 * register EntryHi) on a CPU. The bit number comes from the CPU's cpu_index(). 74 * Even though these bitmaps contain the bits for all CPUs, the bits that 75 * correspond to the bits belonging to the CPUs sharing a TLB can only be 76 * manipulated while holding that TLB's lock. Atomic ops must be used to 77 * update them since multiple CPUs may be changing different sets of bits at 78 * same time but these sets never overlap. 79 * 80 * When a change to the local TLB may require a change in the TLB's of other 81 * CPUs, we try to avoid sending an IPI if at all possible. For instance, if 82 * we are updating a PTE and that PTE previously was invalid and therefore 83 * couldn't support an active mapping, there's no need for an IPI since there 84 * can't be a TLB entry to invalidate. The other case is when we change a PTE 85 * to be modified we just update the local TLB. If another TLB has a stale 86 * entry, a TLB MOD exception will be raised and that will cause the local TLB 87 * to be updated. 88 * 89 * We never need to update a non-local TLB if the pmap doesn't have a valid 90 * ASID for that TLB. If it does have a valid ASID but isn't current "onproc" 91 * we simply reset its ASID for that TLB and then when it goes "onproc" it 92 * will allocate a new ASID and any existing TLB entries will be orphaned. 93 * Only in the case that pmap has an "onproc" ASID do we actually have to send 94 * an IPI. 95 * 96 * Once we determined we must send an IPI to shootdown a TLB, we need to send 97 * it to one of CPUs that share that TLB. We choose the lowest numbered CPU 98 * that has one of the pmap's ASID "onproc". In reality, any CPU sharing that 99 * TLB would do, but interrupting an active CPU seems best. 100 * 101 * A TLB might have multiple shootdowns active concurrently. The shootdown 102 * logic compresses these into a few cases: 103 * 0) nobody needs to have its TLB entries invalidated 104 * 1) one ASID needs to have its TLB entries invalidated 105 * 2) more than one ASID needs to have its TLB entries invalidated 106 * 3) the kernel needs to have its TLB entries invalidated 107 * 4) the kernel and one or more ASID need their TLB entries invalidated. 108 * 109 * And for each case we do: 110 * 0) nothing, 111 * 1) if that ASID is still "onproc", we invalidate the TLB entries for 112 * that single ASID. If not, just reset the pmap's ASID to invalidate 113 * and let it allocate a new ASID the next time it goes "onproc", 114 * 2) we reinitialize the ASID space (preserving any "onproc" ASIDs) and 115 * invalidate all non-wired non-global TLB entries, 116 * 3) we invalidate all of the non-wired global TLB entries, 117 * 4) we reinitialize the ASID space (again preserving any "onproc" ASIDs) 118 * invalidate all non-wired TLB entries. 119 * 120 * As you can see, shootdowns are not concerned with addresses, just address 121 * spaces. Since the number of TLB entries is usually quite small, this avoids 122 * a lot of overhead for not much gain. 123 */ 124 125 #define __PMAP_PRIVATE 126 127 #include "opt_multiprocessor.h" 128 129 #include <sys/param.h> 130 #include <sys/systm.h> 131 #include <sys/proc.h> 132 #include <sys/mutex.h> 133 #include <sys/atomic.h> 134 #include <sys/kernel.h> /* for cold */ 135 #include <sys/cpu.h> 136 137 #include <uvm/uvm.h> 138 139 static kmutex_t pmap_tlb0_mutex __cacheline_aligned; 140 141 #define IFCONSTANT(x) (__builtin_constant_p((x)) ? (x) : 0) 142 143 struct pmap_tlb_info pmap_tlb0_info = { 144 .ti_name = "tlb0", 145 .ti_asid_hint = KERNEL_PID + 1, 146 #ifdef PMAP_TLB_NUM_PIDS 147 .ti_asid_max = IFCONSTANT(PMAP_TLB_NUM_PIDS - 1), 148 .ti_asids_free = IFCONSTANT(PMAP_TLB_NUM_PIDS - (KERNEL_PID + 1)), 149 #endif 150 .ti_asid_bitmap[0] = (2 << KERNEL_PID) - 1, 151 #ifdef PMAP_TLB_WIRED_UPAGES 152 .ti_wired = PMAP_TLB_WIRED_UPAGES, 153 #endif 154 .ti_lock = &pmap_tlb0_mutex, 155 .ti_pais = LIST_HEAD_INITIALIZER(pmap_tlb0_info.ti_pais), 156 #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1 157 .ti_tlbinvop = TLBINV_NOBODY, 158 #endif 159 }; 160 161 #undef IFCONSTANT 162 163 #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1 164 struct pmap_tlb_info *pmap_tlbs[PMAP_TLB_MAX] = { 165 [0] = &pmap_tlb0_info, 166 }; 167 u_int pmap_ntlbs = 1; 168 #endif 169 170 #define __BITMAP_SET(bm, n) \ 171 ((bm)[(n) / (8*sizeof(bm[0]))] |= 1LU << ((n) % (8*sizeof(bm[0])))) 172 #define __BITMAP_CLR(bm, n) \ 173 ((bm)[(n) / (8*sizeof(bm[0]))] &= ~(1LU << ((n) % (8*sizeof(bm[0]))))) 174 #define __BITMAP_ISSET_P(bm, n) \ 175 (((bm)[(n) / (8*sizeof(bm[0]))] & (1LU << ((n) % (8*sizeof(bm[0]))))) != 0) 176 177 #define TLBINFO_ASID_MARK_USED(ti, asid) \ 178 __BITMAP_SET((ti)->ti_asid_bitmap, (asid)) 179 #define TLBINFO_ASID_INUSE_P(ti, asid) \ 180 __BITMAP_ISSET_P((ti)->ti_asid_bitmap, (asid)) 181 182 static void 183 pmap_pai_check(struct pmap_tlb_info *ti) 184 { 185 #ifdef DIAGNOSTIC 186 struct pmap_asid_info *pai; 187 LIST_FOREACH(pai, &ti->ti_pais, pai_link) { 188 KASSERT(pai != NULL); 189 KASSERT(PAI_PMAP(pai, ti) != pmap_kernel()); 190 KASSERT(pai->pai_asid > KERNEL_PID); 191 KASSERT(TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)); 192 } 193 #endif 194 } 195 196 #ifdef MULTIPROCESSOR 197 static inline bool 198 pmap_tlb_intersecting_active_p(pmap_t pm, struct pmap_tlb_info *ti) 199 { 200 #if PMAP_TLB_MAX == 1 201 return !kcpuset_iszero(pm->pm_active); 202 #else 203 return kcpuset_intersecting_p(pm->pm_active, ti->ti_kcpuset); 204 #endif 205 } 206 207 static inline bool 208 pmap_tlb_intersecting_onproc_p(pmap_t pm, struct pmap_tlb_info *ti) 209 { 210 #if PMAP_TLB_MAX == 1 211 return !kcpuset_iszero(pm->pm_onproc); 212 #else 213 return kcpuset_intersecting_p(pm->pm_onproc, ti->ti_kcpuset); 214 #endif 215 } 216 #endif 217 218 static inline void 219 pmap_pai_reset(struct pmap_tlb_info *ti, struct pmap_asid_info *pai, 220 struct pmap *pm) 221 { 222 /* 223 * We must have an ASID but it must not be onproc (on a processor). 224 */ 225 KASSERT(pai->pai_asid > KERNEL_PID); 226 #if defined(MULTIPROCESSOR) 227 KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti)); 228 #endif 229 LIST_REMOVE(pai, pai_link); 230 #ifdef DIAGNOSTIC 231 pai->pai_link.le_prev = NULL; /* tagged as unlinked */ 232 #endif 233 /* 234 * Note that we don't mark the ASID as not in use in the TLB's ASID 235 * bitmap (thus it can't be allocated until the ASID space is exhausted 236 * and therefore reinitialized). We don't want to flush the TLB for 237 * entries belonging to this ASID so we will let natural TLB entry 238 * replacement flush them out of the TLB. Any new entries for this 239 * pmap will need a new ASID allocated. 240 */ 241 pai->pai_asid = 0; 242 243 #if defined(MULTIPROCESSOR) 244 /* 245 * The bits in pm_active belonging to this TLB can only be changed 246 * while this TLB's lock is held. 247 */ 248 #if PMAP_TLB_MAX == 1 249 kcpuset_zero(pm->pm_active); 250 #else 251 kcpuset_atomicly_remove(pm->pm_active, ti->ti_kcpuset); 252 #endif 253 #endif /* MULTIPROCESSOR */ 254 } 255 256 void 257 pmap_tlb_info_evcnt_attach(struct pmap_tlb_info *ti) 258 { 259 #if defined(MULTIPROCESSOR) 260 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_desired, 261 EVCNT_TYPE_MISC, NULL, 262 ti->ti_name, "icache syncs desired"); 263 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_asts, 264 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired, 265 ti->ti_name, "icache sync asts"); 266 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_all, 267 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_asts, 268 ti->ti_name, "icache full syncs"); 269 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_pages, 270 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_asts, 271 ti->ti_name, "icache pages synced"); 272 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_duplicate, 273 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired, 274 ti->ti_name, "icache dup pages skipped"); 275 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_deferred, 276 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired, 277 ti->ti_name, "icache pages deferred"); 278 #endif /* MULTIPROCESSOR */ 279 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_asid_reinits, 280 EVCNT_TYPE_MISC, NULL, 281 ti->ti_name, "asid pool reinit"); 282 } 283 284 void 285 pmap_tlb_info_init(struct pmap_tlb_info *ti) 286 { 287 #if defined(MULTIPROCESSOR) 288 #if PMAP_TLB_MAX == 1 289 KASSERT(ti == &pmap_tlb0_info); 290 #else 291 if (ti != &pmap_tlb0_info) { 292 KASSERT(pmap_ntlbs < PMAP_TLB_MAX); 293 294 KASSERT(pmap_tlbs[pmap_ntlbs] == NULL); 295 296 ti->ti_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SCHED); 297 ti->ti_asid_bitmap[0] = (2 << KERNEL_PID) - 1; 298 ti->ti_asid_hint = KERNEL_PID + 1; 299 ti->ti_asid_max = pmap_tlbs[0]->ti_asid_max; 300 ti->ti_asids_free = ti->ti_asid_max - KERNEL_PID; 301 ti->ti_tlbinvop = TLBINV_NOBODY, 302 ti->ti_victim = NULL; 303 kcpuset_create(&ti->ti_kcpuset, true); 304 ti->ti_index = pmap_ntlbs++; 305 ti->ti_wired = 0; 306 pmap_tlbs[ti->ti_index] = ti; 307 snprintf(ti->ti_name, sizeof(ti->ti_name), "tlb%u", 308 ti->ti_index); 309 pmap_tlb_info_evcnt_attach(ti); 310 return; 311 } 312 #endif 313 #endif /* MULTIPROCESSOR */ 314 KASSERT(ti == &pmap_tlb0_info); 315 mutex_init(ti->ti_lock, MUTEX_DEFAULT, IPL_SCHED); 316 #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1 317 kcpuset_create(&ti->ti_kcpuset, true); 318 kcpuset_set(&ti->ti_kcpuset, cpu_index(curcpu())); 319 #endif 320 if (ti->ti_asid_max == 0) { 321 ti->ti_asid_max = pmap_md_tlb_asid_max(); 322 ti->ti_asids_free = ti->ti_asid_max - (KERNEL_PID + 1); 323 } 324 325 KASSERT(ti->ti_asid_max < sizeof(ti->ti_asid_bitmap)*8); 326 } 327 328 #if defined(MULTIPROCESSOR) 329 void 330 pmap_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci) 331 { 332 KASSERT(!CPU_IS_PRIMARY(ci)); 333 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 334 KASSERT(cold); 335 336 TLBINFO_LOCK(ti); 337 #if PMAP_TLB_MAX > 1 338 kcpuset_set(ti->ti_kcpuset, cpu_index(ci)); 339 #endif 340 cpu_set_tlb_info(ci, ti); 341 342 /* 343 * Do any MD tlb info init. 344 */ 345 pmap_md_tlb_info_attach(ti, ci); 346 347 /* 348 * The kernel pmap uses the kcpuset_running set so it's always 349 * up-to-date. 350 */ 351 TLBINFO_UNLOCK(ti); 352 } 353 #endif /* MULTIPROCESSOR */ 354 355 #ifdef DIAGNOSTIC 356 static size_t 357 pmap_tlb_asid_count(struct pmap_tlb_info *ti) 358 { 359 size_t count = 0; 360 for (tlb_asid_t asid = 1; asid <= ti->ti_asid_max; asid++) { 361 count += TLBINFO_ASID_INUSE_P(ti, asid); 362 } 363 return count; 364 } 365 #endif 366 367 static void 368 pmap_tlb_asid_reinitialize(struct pmap_tlb_info *ti, enum tlb_invalidate_op op) 369 { 370 const size_t asid_bitmap_words = 371 ti->ti_asid_max / (8 * sizeof(ti->ti_asid_bitmap[0])); 372 373 pmap_pai_check(ti); 374 375 /* 376 * First, clear the ASID bitmap (except for ASID 0 which belongs 377 * to the kernel). 378 */ 379 ti->ti_asids_free = ti->ti_asid_max - KERNEL_PID; 380 ti->ti_asid_hint = KERNEL_PID + 1; 381 ti->ti_asid_bitmap[0] = (2 << KERNEL_PID) - 1; 382 for (size_t word = 1; word <= asid_bitmap_words; word++) { 383 ti->ti_asid_bitmap[word] = 0; 384 } 385 386 switch (op) { 387 #if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN) 388 case TLBINV_ALL: 389 tlb_invalidate_all(); 390 break; 391 case TLBINV_ALLUSER: 392 tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max); 393 break; 394 #endif /* MULTIPROCESSOR && PMAP_NEED_TLB_SHOOTDOWN */ 395 case TLBINV_NOBODY: { 396 /* 397 * If we are just reclaiming ASIDs in the TLB, let's go find 398 * what ASIDs are in use in the TLB. Since this is a 399 * semi-expensive operation, we don't want to do it too often. 400 * So if more half of the ASIDs are in use, we don't have 401 * enough free ASIDs so invalidate the TLB entries with ASIDs 402 * and clear the ASID bitmap. That will force everyone to 403 * allocate a new ASID. 404 */ 405 #if !defined(MULTIPROCESSOR) || defined(PMAP_NEED_TLB_SHOOTDOWN) 406 pmap_tlb_asid_check(); 407 const u_int asids_found = tlb_record_asids(ti->ti_asid_bitmap); 408 pmap_tlb_asid_check(); 409 KASSERT(asids_found == pmap_tlb_asid_count(ti)); 410 if (__predict_false(asids_found >= ti->ti_asid_max / 2)) { 411 tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max); 412 #else /* MULTIPROCESSOR && !PMAP_NEED_TLB_SHOOTDOWN */ 413 /* 414 * For those systems (PowerPC) that don't need require 415 * cross cpu TLB shootdowns, we have to invalidate the 416 * entire TLB because we can't record the ASIDs in use 417 * on the other CPUs. This is hopefully cheaper than 418 * than trying to use an IPI to record all the ASIDs 419 * on all the CPUs (which would be a synchronization 420 * nightmare). 421 */ 422 tlb_invalidate_all(); 423 #endif /* MULTIPROCESSOR && !PMAP_NEED_TLB_SHOOTDOWN */ 424 ti->ti_asid_bitmap[0] = (2 << KERNEL_PID) - 1; 425 for (size_t word = 1; 426 word <= asid_bitmap_words; 427 word++) { 428 ti->ti_asid_bitmap[word] = 0; 429 } 430 #if !defined(MULTIPROCESSOR) || defined(PMAP_NEED_TLB_SHOOTDOWN) 431 } else { 432 ti->ti_asids_free -= asids_found; 433 } 434 #endif /* !MULTIPROCESSOR || PMAP_NEED_TLB_SHOOTDOWN */ 435 break; 436 } 437 default: 438 panic("%s: unexpected op %d", __func__, op); 439 } 440 441 /* 442 * Now go through the active ASIDs. If the ASID is on a processor or 443 * we aren't invalidating all ASIDs and the TLB has an entry owned by 444 * that ASID, mark it as in use. Otherwise release the ASID. 445 */ 446 struct pmap_asid_info *pai, *next; 447 for (pai = LIST_FIRST(&ti->ti_pais); pai != NULL; pai = next) { 448 struct pmap * const pm = PAI_PMAP(pai, ti); 449 next = LIST_NEXT(pai, pai_link); 450 KASSERT(pm != pmap_kernel()); 451 KASSERT(pai->pai_asid > KERNEL_PID); 452 #if defined(MULTIPROCESSOR) 453 if (pmap_tlb_intersecting_onproc_p(pm, ti)) { 454 if (!TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) { 455 TLBINFO_ASID_MARK_USED(ti, pai->pai_asid); 456 ti->ti_asids_free--; 457 } 458 continue; 459 } 460 #endif /* MULTIPROCESSOR */ 461 if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) { 462 KASSERT(op == TLBINV_NOBODY); 463 } else { 464 pmap_pai_reset(ti, pai, pm); 465 } 466 } 467 #ifdef DIAGNOSTIC 468 size_t free_count = ti->ti_asid_max - pmap_tlb_asid_count(ti); 469 if (free_count != ti->ti_asids_free) 470 panic("%s: bitmap error: %zu != %u", 471 __func__, free_count, ti->ti_asids_free); 472 #endif 473 } 474 475 #if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN) 476 #if PMAP_MAX_TLB == 1 477 #error shootdown not required for single TLB systems 478 #endif 479 void 480 pmap_tlb_shootdown_process(void) 481 { 482 struct cpu_info * const ci = curcpu(); 483 struct pmap_tlb_info * const ti = cpu_tlb_info(ci); 484 #ifdef DIAGNOSTIC 485 struct pmap * const pm = curlwp->l_proc->p_vmspace->vm_map.pmap; 486 #endif 487 488 KASSERT(cpu_intr_p()); 489 KASSERTMSG(ci->ci_cpl >= IPL_SCHED, 490 "%s: cpl (%d) < IPL_SCHED (%d)", 491 __func__, ci->ci_cpl, IPL_SCHED); 492 493 TLBINFO_LOCK(ti); 494 495 switch (ti->ti_tlbinvop) { 496 case TLBINV_ONE: { 497 /* 498 * We only need to invalidate one user ASID. 499 */ 500 struct pmap_asid_info * const pai = PMAP_PAI(ti->ti_victim, ti); 501 KASSERT(ti->ti_victim != pmap_kernel()); 502 if (!pmap_tlb_intersecting_onproc_p(ti_victim->pm_onproc, ti)) { 503 /* 504 * The victim is an active pmap so we will just 505 * invalidate its TLB entries. 506 */ 507 KASSERT(pai->pai_asid > KERNEL_PID); 508 pmap_tlb_asid_check(); 509 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid); 510 pmap_tlb_asid_check(); 511 } else if (pai->pai_asid) { 512 /* 513 * The victim is no longer an active pmap for this TLB. 514 * So simply clear its ASID and when pmap_activate is 515 * next called for this pmap, it will allocate a new 516 * ASID. 517 */ 518 KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti)); 519 pmap_pai_reset(ti, pai, PAI_PMAP(pai, ti)); 520 } 521 break; 522 } 523 case TLBINV_ALLUSER: 524 /* 525 * Flush all user TLB entries. 526 */ 527 pmap_tlb_asid_reinitialize(ti, TLBINV_ALLUSER); 528 break; 529 case TLBINV_ALLKERNEL: 530 /* 531 * We need to invalidate all global TLB entries. 532 */ 533 pmap_tlb_asid_check(); 534 tlb_invalidate_globals(); 535 pmap_tlb_asid_check(); 536 break; 537 case TLBINV_ALL: 538 /* 539 * Flush all the TLB entries (user and kernel). 540 */ 541 pmap_tlb_asid_reinitialize(ti, TLBINV_ALL); 542 break; 543 case TLBINV_NOBODY: 544 /* 545 * Might be spurious or another SMT CPU sharing this TLB 546 * could have already done the work. 547 */ 548 break; 549 } 550 551 /* 552 * Indicate we are done with shutdown event. 553 */ 554 ti->ti_victim = NULL; 555 ti->ti_tlbinvop = TLBINV_NOBODY; 556 TLBINFO_UNLOCK(ti); 557 } 558 559 /* 560 * This state machine could be encoded into an array of integers but since all 561 * the values fit in 3 bits, the 5 entry "table" fits in a 16 bit value which 562 * can be loaded in a single instruction. 563 */ 564 #define TLBINV_MAP(op, nobody, one, alluser, allkernel, all) \ 565 (((( (nobody) << 3*TLBINV_NOBODY) \ 566 | ( (one) << 3*TLBINV_ONE) \ 567 | ( (alluser) << 3*TLBINV_ALLUSER) \ 568 | ((allkernel) << 3*TLBINV_ALLKERNEL) \ 569 | ( (all) << 3*TLBINV_ALL)) >> 3*(op)) & 7) 570 571 #define TLBINV_USER_MAP(op) \ 572 TLBINV_MAP(op, TLBINV_ONE, TLBINV_ALLUSER, TLBINV_ALLUSER, \ 573 TLBINV_ALL, TLBINV_ALL) 574 575 #define TLBINV_KERNEL_MAP(op) \ 576 TLBINV_MAP(op, TLBINV_ALLKERNEL, TLBINV_ALL, TLBINV_ALL, \ 577 TLBINV_ALLKERNEL, TLBINV_ALL) 578 579 bool 580 pmap_tlb_shootdown_bystanders(pmap_t pm) 581 { 582 /* 583 * We don't need to deal our own TLB. 584 */ 585 kcpuset_t *pm_active; 586 587 kcpuset_clone(&pm_active, pm->pm_active); 588 kcpuset_atomicly_remove(pm->pm_active, 589 cpu_tlb_info(curcpu())->ti_kcpuset); 590 const bool kernel_p = (pm == pmap_kernel()); 591 bool ipi_sent = false; 592 593 /* 594 * If pm_active gets more bits set, then it's after all our changes 595 * have been made so they will already be cognizant of them. 596 */ 597 598 for (size_t i = 0; !kcpuset_iszero(pm_active); i++) { 599 KASSERT(i < pmap_ntlbs); 600 struct pmap_tlb_info * const ti = pmap_tlbs[i]; 601 KASSERT(tlbinfo_index(ti) == i); 602 /* 603 * Skip this TLB if there are no active mappings for it. 604 */ 605 if (!kcpuset_intersecting_p(pm_active, ti->ti_kcpuset)) 606 continue; 607 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 608 kcpuset_remove(pm_active, ti->ti_kcpuset); 609 TLBINFO_LOCK(ti); 610 if (pmap_tlb_intersecting_onproc_p(pm, ti)) { 611 cpuid_t j = kcpuset_ffs_intersecting(pm->pm_onproc, 612 ti->ti_kcpuset); 613 if (kernel_p) { 614 ti->ti_tlbinvop = 615 TLBINV_KERNEL_MAP(ti->ti_tlbinvop); 616 ti->ti_victim = NULL; 617 } else { 618 KASSERT(pai->pai_asid); 619 if (__predict_false(ti->ti_victim == pm)) { 620 KASSERT(ti->ti_tlbinvop == TLBINV_ONE); 621 /* 622 * We still need to invalidate this one 623 * ASID so there's nothing to change. 624 */ 625 } else { 626 ti->ti_tlbinvop = 627 TLBINV_USER_MAP(ti->ti_tlbinvop); 628 if (ti->ti_tlbinvop == TLBINV_ONE) 629 ti->ti_victim = pm; 630 else 631 ti->ti_victim = NULL; 632 } 633 } 634 TLBINFO_UNLOCK(ti); 635 /* 636 * Now we can send out the shootdown IPIs to a CPU 637 * that shares this TLB and is currently using this 638 * pmap. That CPU will process the IPI and do the 639 * all the work. Any other CPUs sharing that TLB 640 * will take advantage of that work. pm_onproc might 641 * change now that we have released the lock but we 642 * can tolerate spurious shootdowns. 643 */ 644 cpu_send_ipi(cpu_lookup(j), IPI_SHOOTDOWN); 645 ipi_sent = true; 646 continue; 647 } 648 if (!pmap_tlb_intersecting_active_p(pm, ti)) { 649 /* 650 * If this pmap has an ASID assigned but it's not 651 * currently running, nuke its ASID. Next time the 652 * pmap is activated, it will allocate a new ASID. 653 * And best of all, we avoid an IPI. 654 */ 655 KASSERT(!kernel_p); 656 pmap_pai_reset(ti, pai, pm); 657 //ti->ti_evcnt_lazy_shots.ev_count++; 658 } 659 TLBINFO_UNLOCK(ti); 660 } 661 662 kcpuset_destroy(pm_active); 663 664 return ipi_sent; 665 } 666 #endif /* MULTIPROCESSOR && PMAP_NEED_TLB_SHOOTDOWN */ 667 668 #ifndef PMAP_TLB_HWPAGEWALKER 669 int 670 pmap_tlb_update_addr(pmap_t pm, vaddr_t va, pt_entry_t pt_entry, u_int flags) 671 { 672 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 673 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 674 int rv = -1; 675 676 KASSERT(kpreempt_disabled()); 677 678 TLBINFO_LOCK(ti); 679 if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) { 680 pmap_tlb_asid_check(); 681 rv = tlb_update_addr(va, pai->pai_asid, pt_entry, 682 (flags & PMAP_TLB_INSERT) != 0); 683 pmap_tlb_asid_check(); 684 } 685 #if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN) 686 pm->pm_shootdown_pending = (flags & PMAP_TLB_NEED_IPI) != 0; 687 #endif 688 TLBINFO_UNLOCK(ti); 689 690 return rv; 691 } 692 #endif /* !PMAP_TLB_HWPAGEWALKER */ 693 694 void 695 pmap_tlb_invalidate_addr(pmap_t pm, vaddr_t va) 696 { 697 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 698 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 699 700 KASSERT(kpreempt_disabled()); 701 702 TLBINFO_LOCK(ti); 703 if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) { 704 pmap_tlb_asid_check(); 705 tlb_invalidate_addr(va, pai->pai_asid); 706 pmap_tlb_asid_check(); 707 } 708 #if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN) 709 pm->pm_shootdown_pending = 1; 710 #endif 711 TLBINFO_UNLOCK(ti); 712 } 713 714 static inline void 715 pmap_tlb_asid_alloc(struct pmap_tlb_info *ti, pmap_t pm, 716 struct pmap_asid_info *pai) 717 { 718 /* 719 * We shouldn't have an ASID assigned, and thusly must not be onproc 720 * nor active. 721 */ 722 KASSERT(pm != pmap_kernel()); 723 KASSERT(pai->pai_asid == 0); 724 KASSERT(pai->pai_link.le_prev == NULL); 725 #if defined(MULTIPROCESSOR) 726 KASSERT(pmap_tlb_intersecting_onproc_p(pm, ti)); 727 KASSERT(pmap_tlb_intersecting_active_p(pm, ti)); 728 #endif 729 KASSERT(ti->ti_asids_free > 0); 730 KASSERT(ti->ti_asid_hint <= ti->ti_asid_max); 731 732 /* 733 * Let's see if the hinted ASID is free. If not search for 734 * a new one. 735 */ 736 if (__predict_false(TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint))) { 737 #ifdef DIAGNOSTIC 738 const size_t words = __arraycount(ti->ti_asid_bitmap); 739 #endif 740 const size_t nbpw = 8 * sizeof(ti->ti_asid_bitmap[0]); 741 for (size_t i = 0; i < ti->ti_asid_hint / nbpw; i++) { 742 KASSERT(~ti->ti_asid_bitmap[i] == 0); 743 } 744 for (size_t i = ti->ti_asid_hint / nbpw;; i++) { 745 KASSERT(i < words); 746 /* 747 * ffs wants to find the first bit set while we want 748 * to find the first bit cleared. 749 */ 750 u_long bits = ~ti->ti_asid_bitmap[i]; 751 if (__predict_true(bits)) { 752 u_int n = 0; 753 if ((bits & 0xffffffff) == 0) { 754 bits = (bits >> 31) >> 1; 755 KASSERT(bits); 756 n += 32; 757 } 758 n += ffs(bits) - 1; 759 KASSERT(n < nbpw); 760 ti->ti_asid_hint = n + i * nbpw; 761 break; 762 } 763 } 764 KASSERT(ti->ti_asid_hint > KERNEL_PID); 765 KASSERT(TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint-1)); 766 KASSERT(!TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint)); 767 } 768 769 /* 770 * The hint contains our next ASID so take it and advance the hint. 771 * Mark it as used and insert the pai into the list of active asids. 772 * There is also one less asid free in this TLB. 773 */ 774 pai->pai_asid = ti->ti_asid_hint++; 775 TLBINFO_ASID_MARK_USED(ti, pai->pai_asid); 776 LIST_INSERT_HEAD(&ti->ti_pais, pai, pai_link); 777 ti->ti_asids_free--; 778 779 #if defined(MULTIPROCESSOR) 780 /* 781 * Mark that we now have an active ASID for all CPUs sharing this TLB. 782 * The bits in pm_active belonging to this TLB can only be changed 783 * while this TLBs lock is held. 784 */ 785 #if PMAP_TLB_MAX == 1 786 kcpuset_copy(pm->pm_active, kcpuset_running); 787 #else 788 kcpuset_atomicly_merge(pm->pm_active, ti->ti_kcpuset); 789 #endif 790 #endif 791 } 792 793 /* 794 * Acquire a TLB address space tag (called ASID or TLBPID) and return it. 795 * ASID might have already been previously acquired. 796 */ 797 void 798 pmap_tlb_asid_acquire(pmap_t pm, struct lwp *l) 799 { 800 struct cpu_info * const ci = l->l_cpu; 801 struct pmap_tlb_info * const ti = cpu_tlb_info(ci); 802 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 803 804 KASSERT(kpreempt_disabled()); 805 806 /* 807 * Kernels use a fixed ASID and thus doesn't need to acquire one. 808 */ 809 if (pm == pmap_kernel()) 810 return; 811 812 TLBINFO_LOCK(ti); 813 KASSERT(pai->pai_asid <= KERNEL_PID || pai->pai_link.le_prev != NULL); 814 KASSERT(pai->pai_asid > KERNEL_PID || pai->pai_link.le_prev == NULL); 815 pmap_pai_check(ti); 816 if (__predict_false(!PMAP_PAI_ASIDVALID_P(pai, ti))) { 817 /* 818 * If we've run out ASIDs, reinitialize the ASID space. 819 */ 820 if (__predict_false(tlbinfo_noasids_p(ti))) { 821 KASSERT(l == curlwp); 822 pmap_tlb_asid_reinitialize(ti, TLBINV_NOBODY); 823 } 824 825 /* 826 * Get an ASID. 827 */ 828 pmap_tlb_asid_alloc(ti, pm, pai); 829 } 830 831 if (l == curlwp) { 832 #if defined(MULTIPROCESSOR) 833 /* 834 * The bits in pm_onproc belonging to this TLB can only 835 * be changed while this TLBs lock is held unless atomic 836 * operations are used. 837 */ 838 kcpuset_atomic_set(pm->pm_onproc, cpu_index(ci)); 839 #endif 840 ci->ci_pmap_asid_cur = pai->pai_asid; 841 tlb_set_asid(pai->pai_asid); 842 pmap_tlb_asid_check(); 843 } else { 844 printf("%s: l (%p) != curlwp %p\n", __func__, l, curlwp); 845 } 846 TLBINFO_UNLOCK(ti); 847 } 848 849 void 850 pmap_tlb_asid_deactivate(pmap_t pm) 851 { 852 KASSERT(kpreempt_disabled()); 853 #if defined(MULTIPROCESSOR) 854 /* 855 * The kernel pmap is aways onproc and active and must never have 856 * those bits cleared. If pmap_remove_all was called, it has already 857 * deactivated the pmap and thusly onproc will be 0 so there's nothing 858 * to do. 859 */ 860 if (pm != pmap_kernel() && pm->pm_onproc != 0) { 861 struct cpu_info * const ci = curcpu(); 862 KASSERT(!cpu_intr_p()); 863 KASSERTMSG(kcpuset_isset(pm->pm_onproc, cpu_index(ci)), 864 "%s: pmap %p onproc %p doesn't include cpu %d (%p)", 865 __func__, pm, pm->pm_onproc, cpu_index(ci), ci); 866 /* 867 * The bits in pm_onproc that belong to this TLB can 868 * be changed while this TLBs lock is not held as long 869 * as we use atomic ops. 870 */ 871 kcpuset_atomic_clear(pm->pm_onproc, cpu_index(ci)); 872 } 873 #elif defined(DEBUG) 874 curcpu()->ci_pmap_asid_cur = 0; 875 tlb_set_asid(0); 876 pmap_tlb_asid_check(); 877 #endif 878 } 879 880 void 881 pmap_tlb_asid_release_all(struct pmap *pm) 882 { 883 KASSERT(pm != pmap_kernel()); 884 #if defined(MULTIPROCESSOR) 885 KASSERT(kcpuset_iszero(pm->pm_onproc)); 886 #if PMAP_TLB_MAX > 1 887 for (u_int i = 0; !kcpuset_iszero(pm->pm_active); i++) { 888 KASSERT(i < pmap_ntlbs); 889 struct pmap_tlb_info * const ti = pmap_tlbs[i]; 890 #else 891 struct pmap_tlb_info * const ti = &pmap_tlb0_info; 892 #endif 893 if (!pmap_tlb_intersecting_onproc_p(pm, ti)) { 894 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 895 TLBINFO_LOCK(ti); 896 KASSERT(ti->ti_victim != pm); 897 pmap_pai_reset(ti, pai, pm); 898 TLBINFO_UNLOCK(ti); 899 } 900 #if PMAP_TLB_MAX > 1 901 } 902 #endif 903 #else 904 /* 905 * Handle the case of an UP kernel which only has, at most, one ASID. 906 * If the pmap has an ASID allocated, free it. 907 */ 908 struct pmap_tlb_info * const ti = &pmap_tlb0_info; 909 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 910 TLBINFO_LOCK(ti); 911 if (pai->pai_asid > KERNEL_PID) { 912 pmap_pai_reset(ti, pai, pm); 913 } 914 TLBINFO_UNLOCK(ti); 915 #endif /* MULTIPROCESSOR */ 916 } 917 918 void 919 pmap_tlb_asid_check(void) 920 { 921 #ifdef DEBUG 922 kpreempt_disable(); 923 const tlb_asid_t asid = tlb_get_asid(); 924 KDASSERTMSG(asid == curcpu()->ci_pmap_asid_cur, 925 "%s: asid (%#x) != current asid (%#x)", 926 __func__, asid, curcpu()->ci_pmap_asid_cur); 927 kpreempt_enable(); 928 #endif 929 } 930 931 #ifdef DEBUG 932 void 933 pmap_tlb_check(pmap_t pm, bool (*func)(void *, vaddr_t, tlb_asid_t, pt_entry_t)) 934 { 935 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 936 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 937 TLBINFO_LOCK(ti); 938 if (pm == pmap_kernel() || pai->pai_asid > KERNEL_PID) 939 tlb_walk(pm, func); 940 TLBINFO_UNLOCK(ti); 941 } 942 #endif /* DEBUG */ 943