1 /* $NetBSD: pmap_tlb.c,v 1.8 2014/04/03 14:46:25 matt Exp $ */ 2 3 /*- 4 * Copyright (c) 2010 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Matt Thomas at 3am Software Foundry. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 34 __KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.8 2014/04/03 14:46:25 matt Exp $"); 35 36 /* 37 * Manages address spaces in a TLB. 38 * 39 * Normally there is a 1:1 mapping between a TLB and a CPU. However, some 40 * implementations may share a TLB between multiple CPUs (really CPU thread 41 * contexts). This requires the TLB abstraction to be separated from the 42 * CPU abstraction. It also requires that the TLB be locked while doing 43 * TLB activities. 44 * 45 * For each TLB, we track the ASIDs in use in a bitmap and a list of pmaps 46 * that have a valid ASID. 47 * 48 * We allocate ASIDs in increasing order until we have exhausted the supply, 49 * then reinitialize the ASID space, and start allocating again at 1. When 50 * allocating from the ASID bitmap, we skip any ASID who has a corresponding 51 * bit set in the ASID bitmap. Eventually this causes the ASID bitmap to fill 52 * and, when completely filled, a reinitialization of the ASID space. 53 * 54 * To reinitialize the ASID space, the ASID bitmap is reset and then the ASIDs 55 * of non-kernel TLB entries get recorded in the ASID bitmap. If the entries 56 * in TLB consume more than half of the ASID space, all ASIDs are invalidated, 57 * the ASID bitmap is recleared, and the list of pmaps is emptied. Otherwise, 58 * (the normal case), any ASID present in the TLB (even those which are no 59 * longer used by a pmap) will remain active (allocated) and all other ASIDs 60 * will be freed. If the size of the TLB is much smaller than the ASID space, 61 * this algorithm completely avoids TLB invalidation. 62 * 63 * For multiprocessors, we also have to deal TLB invalidation requests from 64 * other CPUs, some of which are dealt with the reinitialization of the ASID 65 * space. Whereas above we keep the ASIDs of those pmaps which have active 66 * TLB entries, this type of reinitialization preserves the ASIDs of any 67 * "onproc" user pmap and all other ASIDs will be freed. We must do this 68 * since we can't change the current ASID. 69 * 70 * Each pmap has two bitmaps: pm_active and pm_onproc. Each bit in pm_active 71 * indicates whether that pmap has an allocated ASID for a CPU. Each bit in 72 * pm_onproc indicates that pmap's ASID is active (equal to the ASID in COP 0 73 * register EntryHi) on a CPU. The bit number comes from the CPU's cpu_index(). 74 * Even though these bitmaps contain the bits for all CPUs, the bits that 75 * correspond to the bits belonging to the CPUs sharing a TLB can only be 76 * manipulated while holding that TLB's lock. Atomic ops must be used to 77 * update them since multiple CPUs may be changing different sets of bits at 78 * same time but these sets never overlap. 79 * 80 * When a change to the local TLB may require a change in the TLB's of other 81 * CPUs, we try to avoid sending an IPI if at all possible. For instance, if 82 * we are updating a PTE and that PTE previously was invalid and therefore 83 * couldn't support an active mapping, there's no need for an IPI since there 84 * can't be a TLB entry to invalidate. The other case is when we change a PTE 85 * to be modified we just update the local TLB. If another TLB has a stale 86 * entry, a TLB MOD exception will be raised and that will cause the local TLB 87 * to be updated. 88 * 89 * We never need to update a non-local TLB if the pmap doesn't have a valid 90 * ASID for that TLB. If it does have a valid ASID but isn't current "onproc" 91 * we simply reset its ASID for that TLB and then when it goes "onproc" it 92 * will allocate a new ASID and any existing TLB entries will be orphaned. 93 * Only in the case that pmap has an "onproc" ASID do we actually have to send 94 * an IPI. 95 * 96 * Once we determined we must send an IPI to shootdown a TLB, we need to send 97 * it to one of CPUs that share that TLB. We choose the lowest numbered CPU 98 * that has one of the pmap's ASID "onproc". In reality, any CPU sharing that 99 * TLB would do, but interrupting an active CPU seems best. 100 * 101 * A TLB might have multiple shootdowns active concurrently. The shootdown 102 * logic compresses these into a few cases: 103 * 0) nobody needs to have its TLB entries invalidated 104 * 1) one ASID needs to have its TLB entries invalidated 105 * 2) more than one ASID needs to have its TLB entries invalidated 106 * 3) the kernel needs to have its TLB entries invalidated 107 * 4) the kernel and one or more ASID need their TLB entries invalidated. 108 * 109 * And for each case we do: 110 * 0) nothing, 111 * 1) if that ASID is still "onproc", we invalidate the TLB entries for 112 * that single ASID. If not, just reset the pmap's ASID to invalidate 113 * and let it allocate a new ASID the next time it goes "onproc", 114 * 2) we reinitialize the ASID space (preserving any "onproc" ASIDs) and 115 * invalidate all non-wired non-global TLB entries, 116 * 3) we invalidate all of the non-wired global TLB entries, 117 * 4) we reinitialize the ASID space (again preserving any "onproc" ASIDs) 118 * invalidate all non-wired TLB entries. 119 * 120 * As you can see, shootdowns are not concerned with addresses, just address 121 * spaces. Since the number of TLB entries is usually quite small, this avoids 122 * a lot of overhead for not much gain. 123 */ 124 125 #define __PMAP_PRIVATE 126 127 #include "opt_multiprocessor.h" 128 129 #include <sys/param.h> 130 #include <sys/systm.h> 131 #include <sys/proc.h> 132 #include <sys/mutex.h> 133 #include <sys/atomic.h> 134 #include <sys/kernel.h> /* for cold */ 135 #include <sys/cpu.h> 136 137 #include <uvm/uvm.h> 138 139 static kmutex_t pmap_tlb0_lock __cacheline_aligned; 140 141 #define IFCONSTANT(x) (__builtin_constant_p((x)) ? (x) : 0) 142 143 struct pmap_tlb_info pmap_tlb0_info = { 144 .ti_name = "tlb0", 145 .ti_asid_hint = KERNEL_PID + 1, 146 #ifdef PMAP_TLB_NUM_PIDS 147 .ti_asid_max = IFCONSTANT(PMAP_TLB_NUM_PIDS - 1), 148 .ti_asids_free = IFCONSTANT(PMAP_TLB_NUM_PIDS - (1 + KERNEL_PID)), 149 #endif 150 .ti_asid_bitmap[0] = (2 << KERNEL_PID) - 1, 151 #ifdef PMAP_TLB_WIRED_UPAGES 152 .ti_wired = PMAP_TLB_WIRED_UPAGES, 153 #endif 154 .ti_lock = &pmap_tlb0_lock, 155 .ti_pais = LIST_HEAD_INITIALIZER(pmap_tlb0_info.ti_pais), 156 #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1 157 .ti_tlbinvop = TLBINV_NOBODY, 158 #endif 159 }; 160 161 #undef IFCONSTANT 162 163 #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1 164 struct pmap_tlb_info *pmap_tlbs[PMAP_TLB_MAX] = { 165 [0] = &pmap_tlb0_info, 166 }; 167 u_int pmap_ntlbs = 1; 168 #endif 169 170 #define __BITMAP_SET(bm, n) \ 171 ((bm)[(n) / (8*sizeof(bm[0]))] |= 1LU << ((n) % (8*sizeof(bm[0])))) 172 #define __BITMAP_CLR(bm, n) \ 173 ((bm)[(n) / (8*sizeof(bm[0]))] &= ~(1LU << ((n) % (8*sizeof(bm[0]))))) 174 #define __BITMAP_ISSET_P(bm, n) \ 175 (((bm)[(n) / (8*sizeof(bm[0]))] & (1LU << ((n) % (8*sizeof(bm[0]))))) != 0) 176 177 #define TLBINFO_ASID_MARK_UNUSED(ti, asid) \ 178 __BITMAP_CLR((ti)->ti_asid_bitmap, (asid)) 179 #define TLBINFO_ASID_MARK_USED(ti, asid) \ 180 __BITMAP_SET((ti)->ti_asid_bitmap, (asid)) 181 #define TLBINFO_ASID_INUSE_P(ti, asid) \ 182 __BITMAP_ISSET_P((ti)->ti_asid_bitmap, (asid)) 183 184 static void 185 pmap_pai_check(struct pmap_tlb_info *ti) 186 { 187 #ifdef DIAGNOSTIC 188 struct pmap_asid_info *pai; 189 LIST_FOREACH(pai, &ti->ti_pais, pai_link) { 190 KASSERT(pai != NULL); 191 KASSERT(PAI_PMAP(pai, ti) != pmap_kernel()); 192 KASSERT(pai->pai_asid > KERNEL_PID); 193 KASSERT(TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)); 194 } 195 #endif 196 } 197 198 #ifdef MULTIPROCESSOR 199 static inline bool 200 pmap_tlb_intersecting_active_p(pmap_t pm, struct pmap_tlb_info *ti) 201 { 202 #if PMAP_TLB_MAX == 1 203 return !kcpuset_iszero(pm->pm_active); 204 #else 205 return kcpuset_intersecting_p(pm->pm_active, ti->ti_kcpuset); 206 #endif 207 } 208 209 static inline bool 210 pmap_tlb_intersecting_onproc_p(pmap_t pm, struct pmap_tlb_info *ti) 211 { 212 #if PMAP_TLB_MAX == 1 213 return !kcpuset_iszero(pm->pm_onproc); 214 #else 215 return kcpuset_intersecting_p(pm->pm_onproc, ti->ti_kcpuset); 216 #endif 217 } 218 #endif 219 220 static inline void 221 pmap_pai_reset(struct pmap_tlb_info *ti, struct pmap_asid_info *pai, 222 struct pmap *pm) 223 { 224 /* 225 * We must have an ASID but it must not be onproc (on a processor). 226 */ 227 KASSERT(pai->pai_asid > KERNEL_PID); 228 #if defined(MULTIPROCESSOR) 229 KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti)); 230 #endif 231 LIST_REMOVE(pai, pai_link); 232 #ifdef DIAGNOSTIC 233 pai->pai_link.le_prev = NULL; /* tagged as unlinked */ 234 #endif 235 /* 236 * If the platform has a cheap way to flush ASIDs then free the ASID 237 * back into the pool. On multiprocessor systems, we will flush the 238 * ASID from the TLB when it's allocated. That way we know the flush 239 * was always done in the correct TLB space. On uniprocessor systems, 240 * just do the flush now since we know that it has been used. This has 241 * a bit less overhead. Either way, this will mean that we will only 242 * need to flush all ASIDs if all ASIDs are in use and we need to 243 * allocate a new one. 244 */ 245 if (PMAP_TLB_FLUSH_ASID_ON_RESET) { 246 #ifndef MULTIPROCESSOR 247 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid); 248 #endif 249 if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) { 250 TLBINFO_ASID_MARK_UNUSED(ti, pai->pai_asid); 251 ti->ti_asids_free++; 252 } 253 } 254 /* 255 * Note that we don't mark the ASID as not in use in the TLB's ASID 256 * bitmap (thus it can't be allocated until the ASID space is exhausted 257 * and therefore reinitialized). We don't want to flush the TLB for 258 * entries belonging to this ASID so we will let natural TLB entry 259 * replacement flush them out of the TLB. Any new entries for this 260 * pmap will need a new ASID allocated. 261 */ 262 pai->pai_asid = 0; 263 264 #if defined(MULTIPROCESSOR) 265 /* 266 * The bits in pm_active belonging to this TLB can only be changed 267 * while this TLB's lock is held. 268 */ 269 #if PMAP_TLB_MAX == 1 270 kcpuset_zero(pm->pm_active); 271 #else 272 kcpuset_atomicly_remove(pm->pm_active, ti->ti_kcpuset); 273 #endif 274 #endif /* MULTIPROCESSOR */ 275 } 276 277 void 278 pmap_tlb_info_evcnt_attach(struct pmap_tlb_info *ti) 279 { 280 #if defined(MULTIPROCESSOR) 281 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_desired, 282 EVCNT_TYPE_MISC, NULL, 283 ti->ti_name, "icache syncs desired"); 284 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_asts, 285 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired, 286 ti->ti_name, "icache sync asts"); 287 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_all, 288 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_asts, 289 ti->ti_name, "icache full syncs"); 290 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_pages, 291 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_asts, 292 ti->ti_name, "icache pages synced"); 293 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_duplicate, 294 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired, 295 ti->ti_name, "icache dup pages skipped"); 296 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_deferred, 297 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired, 298 ti->ti_name, "icache pages deferred"); 299 #endif /* MULTIPROCESSOR */ 300 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_asid_reinits, 301 EVCNT_TYPE_MISC, NULL, 302 ti->ti_name, "asid pool reinit"); 303 } 304 305 void 306 pmap_tlb_info_init(struct pmap_tlb_info *ti) 307 { 308 #if defined(MULTIPROCESSOR) 309 #if PMAP_TLB_MAX == 1 310 KASSERT(ti == &pmap_tlb0_info); 311 #else 312 if (ti != &pmap_tlb0_info) { 313 KASSERT(pmap_ntlbs < PMAP_TLB_MAX); 314 315 KASSERT(pmap_tlbs[pmap_ntlbs] == NULL); 316 317 ti->ti_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SCHED); 318 ti->ti_asid_bitmap[0] = (2 << KERNEL_PID) - 1; 319 ti->ti_asid_hint = KERNEL_PID + 1; 320 ti->ti_asid_max = pmap_tlbs[0]->ti_asid_max; 321 ti->ti_asids_free = ti->ti_asid_max - KERNEL_PID; 322 ti->ti_tlbinvop = TLBINV_NOBODY, 323 ti->ti_victim = NULL; 324 kcpuset_create(&ti->ti_kcpuset, true); 325 ti->ti_index = pmap_ntlbs++; 326 ti->ti_wired = 0; 327 pmap_tlbs[ti->ti_index] = ti; 328 snprintf(ti->ti_name, sizeof(ti->ti_name), "tlb%u", 329 ti->ti_index); 330 pmap_tlb_info_evcnt_attach(ti); 331 return; 332 } 333 #endif 334 #endif /* MULTIPROCESSOR */ 335 KASSERT(ti == &pmap_tlb0_info); 336 KASSERT(ti->ti_lock == &pmap_tlb0_lock); 337 //printf("ti_lock %p ", ti->ti_lock); 338 mutex_init(ti->ti_lock, MUTEX_DEFAULT, IPL_SCHED); 339 #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1 340 kcpuset_create(&ti->ti_kcpuset, true); 341 kcpuset_set(ti->ti_kcpuset, cpu_index(curcpu())); 342 #endif 343 //printf("asid "); 344 if (ti->ti_asid_max == 0) { 345 ti->ti_asid_max = pmap_md_tlb_asid_max(); 346 ti->ti_asids_free = ti->ti_asid_max - KERNEL_PID; 347 } 348 349 KASSERT(ti->ti_asid_max < sizeof(ti->ti_asid_bitmap)*8); 350 } 351 352 #if defined(MULTIPROCESSOR) 353 void 354 pmap_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci) 355 { 356 KASSERT(!CPU_IS_PRIMARY(ci)); 357 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 358 KASSERT(cold); 359 360 TLBINFO_LOCK(ti); 361 #if PMAP_TLB_MAX > 1 362 kcpuset_set(ti->ti_kcpuset, cpu_index(ci)); 363 cpu_set_tlb_info(ci, ti); 364 #endif 365 366 /* 367 * Do any MD tlb info init. 368 */ 369 pmap_md_tlb_info_attach(ti, ci); 370 371 /* 372 * The kernel pmap uses the kcpuset_running set so it's always 373 * up-to-date. 374 */ 375 TLBINFO_UNLOCK(ti); 376 } 377 #endif /* MULTIPROCESSOR */ 378 379 #ifdef DIAGNOSTIC 380 static size_t 381 pmap_tlb_asid_count(struct pmap_tlb_info *ti) 382 { 383 size_t count = 0; 384 for (tlb_asid_t asid = 1; asid <= ti->ti_asid_max; asid++) { 385 count += TLBINFO_ASID_INUSE_P(ti, asid); 386 } 387 return count; 388 } 389 #endif 390 391 static void 392 pmap_tlb_asid_reinitialize(struct pmap_tlb_info *ti, enum tlb_invalidate_op op) 393 { 394 const size_t asid_bitmap_words = 395 ti->ti_asid_max / (8 * sizeof(ti->ti_asid_bitmap[0])); 396 397 pmap_pai_check(ti); 398 399 ti->ti_evcnt_asid_reinits.ev_count++; 400 401 /* 402 * First, clear the ASID bitmap (except for ASID 0 which belongs 403 * to the kernel). 404 */ 405 ti->ti_asids_free = ti->ti_asid_max - KERNEL_PID; 406 ti->ti_asid_hint = KERNEL_PID + 1; 407 ti->ti_asid_bitmap[0] = (2 << KERNEL_PID) - 1; 408 for (size_t word = 1; word <= asid_bitmap_words; word++) { 409 ti->ti_asid_bitmap[word] = 0; 410 } 411 412 switch (op) { 413 #if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN) 414 case TLBINV_ALL: 415 tlb_invalidate_all(); 416 break; 417 case TLBINV_ALLUSER: 418 tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max); 419 break; 420 #endif /* MULTIPROCESSOR && PMAP_NEED_TLB_SHOOTDOWN */ 421 case TLBINV_NOBODY: { 422 /* 423 * If we are just reclaiming ASIDs in the TLB, let's go find 424 * what ASIDs are in use in the TLB. Since this is a 425 * semi-expensive operation, we don't want to do it too often. 426 * So if more half of the ASIDs are in use, we don't have 427 * enough free ASIDs so invalidate the TLB entries with ASIDs 428 * and clear the ASID bitmap. That will force everyone to 429 * allocate a new ASID. 430 */ 431 #if !defined(MULTIPROCESSOR) || defined(PMAP_NEED_TLB_SHOOTDOWN) 432 pmap_tlb_asid_check(); 433 const u_int asids_found = tlb_record_asids(ti->ti_asid_bitmap); 434 pmap_tlb_asid_check(); 435 KASSERT(asids_found == pmap_tlb_asid_count(ti)); 436 if (__predict_false(asids_found >= ti->ti_asid_max / 2)) { 437 tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max); 438 #else /* MULTIPROCESSOR && !PMAP_NEED_TLB_SHOOTDOWN */ 439 /* 440 * For those systems (PowerPC) that don't need require 441 * cross cpu TLB shootdowns, we have to invalidate the 442 * entire TLB because we can't record the ASIDs in use 443 * on the other CPUs. This is hopefully cheaper than 444 * than trying to use an IPI to record all the ASIDs 445 * on all the CPUs (which would be a synchronization 446 * nightmare). 447 */ 448 tlb_invalidate_all(); 449 #endif /* MULTIPROCESSOR && !PMAP_NEED_TLB_SHOOTDOWN */ 450 ti->ti_asid_bitmap[0] = (2 << KERNEL_PID) - 1; 451 for (size_t word = 1; 452 word <= asid_bitmap_words; 453 word++) { 454 ti->ti_asid_bitmap[word] = 0; 455 } 456 ti->ti_asids_free = ti->ti_asid_max - KERNEL_PID; 457 #if !defined(MULTIPROCESSOR) || defined(PMAP_NEED_TLB_SHOOTDOWN) 458 } else { 459 ti->ti_asids_free -= asids_found; 460 } 461 #endif /* !MULTIPROCESSOR || PMAP_NEED_TLB_SHOOTDOWN */ 462 KASSERTMSG(ti->ti_asids_free <= ti->ti_asid_max, "%u", 463 ti->ti_asids_free); 464 break; 465 } 466 default: 467 panic("%s: unexpected op %d", __func__, op); 468 } 469 470 /* 471 * Now go through the active ASIDs. If the ASID is on a processor or 472 * we aren't invalidating all ASIDs and the TLB has an entry owned by 473 * that ASID, mark it as in use. Otherwise release the ASID. 474 */ 475 struct pmap_asid_info *pai, *next; 476 for (pai = LIST_FIRST(&ti->ti_pais); pai != NULL; pai = next) { 477 struct pmap * const pm = PAI_PMAP(pai, ti); 478 next = LIST_NEXT(pai, pai_link); 479 KASSERT(pm != pmap_kernel()); 480 KASSERT(pai->pai_asid > KERNEL_PID); 481 #if defined(MULTIPROCESSOR) 482 if (pmap_tlb_intersecting_onproc_p(pm, ti)) { 483 if (!TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) { 484 TLBINFO_ASID_MARK_USED(ti, pai->pai_asid); 485 ti->ti_asids_free--; 486 } 487 continue; 488 } 489 #endif /* MULTIPROCESSOR */ 490 if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) { 491 KASSERT(op == TLBINV_NOBODY); 492 } else { 493 pmap_pai_reset(ti, pai, pm); 494 } 495 } 496 #ifdef DIAGNOSTIC 497 size_t free_count __diagused = ti->ti_asid_max - pmap_tlb_asid_count(ti); 498 KASSERTMSG(free_count == ti->ti_asids_free, 499 "bitmap error: %zu != %u", free_count, ti->ti_asids_free); 500 #endif 501 } 502 503 #if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN) 504 #if PMAP_MAX_TLB == 1 505 #error shootdown not required for single TLB systems 506 #endif 507 void 508 pmap_tlb_shootdown_process(void) 509 { 510 struct cpu_info * const ci = curcpu(); 511 struct pmap_tlb_info * const ti = cpu_tlb_info(ci); 512 #ifdef DIAGNOSTIC 513 struct pmap * const pm = curlwp->l_proc->p_vmspace->vm_map.pmap; 514 #endif 515 516 KASSERT(cpu_intr_p()); 517 KASSERTMSG(ci->ci_cpl >= IPL_SCHED, 518 "%s: cpl (%d) < IPL_SCHED (%d)", 519 __func__, ci->ci_cpl, IPL_SCHED); 520 521 TLBINFO_LOCK(ti); 522 523 switch (ti->ti_tlbinvop) { 524 case TLBINV_ONE: { 525 /* 526 * We only need to invalidate one user ASID. 527 */ 528 struct pmap_asid_info * const pai = PMAP_PAI(ti->ti_victim, ti); 529 KASSERT(ti->ti_victim != pmap_kernel()); 530 if (!pmap_tlb_intersecting_onproc_p(ti_victim->pm_onproc, ti)) { 531 /* 532 * The victim is an active pmap so we will just 533 * invalidate its TLB entries. 534 */ 535 KASSERT(pai->pai_asid > KERNEL_PID); 536 pmap_tlb_asid_check(); 537 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid); 538 pmap_tlb_asid_check(); 539 } else if (pai->pai_asid) { 540 /* 541 * The victim is no longer an active pmap for this TLB. 542 * So simply clear its ASID and when pmap_activate is 543 * next called for this pmap, it will allocate a new 544 * ASID. 545 */ 546 KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti)); 547 pmap_pai_reset(ti, pai, PAI_PMAP(pai, ti)); 548 } 549 break; 550 } 551 case TLBINV_ALLUSER: 552 /* 553 * Flush all user TLB entries. 554 */ 555 pmap_tlb_asid_reinitialize(ti, TLBINV_ALLUSER); 556 break; 557 case TLBINV_ALLKERNEL: 558 /* 559 * We need to invalidate all global TLB entries. 560 */ 561 pmap_tlb_asid_check(); 562 tlb_invalidate_globals(); 563 pmap_tlb_asid_check(); 564 break; 565 case TLBINV_ALL: 566 /* 567 * Flush all the TLB entries (user and kernel). 568 */ 569 pmap_tlb_asid_reinitialize(ti, TLBINV_ALL); 570 break; 571 case TLBINV_NOBODY: 572 /* 573 * Might be spurious or another SMT CPU sharing this TLB 574 * could have already done the work. 575 */ 576 break; 577 } 578 579 /* 580 * Indicate we are done with shutdown event. 581 */ 582 ti->ti_victim = NULL; 583 ti->ti_tlbinvop = TLBINV_NOBODY; 584 TLBINFO_UNLOCK(ti); 585 } 586 587 /* 588 * This state machine could be encoded into an array of integers but since all 589 * the values fit in 3 bits, the 5 entry "table" fits in a 16 bit value which 590 * can be loaded in a single instruction. 591 */ 592 #define TLBINV_MAP(op, nobody, one, alluser, allkernel, all) \ 593 (((( (nobody) << 3*TLBINV_NOBODY) \ 594 | ( (one) << 3*TLBINV_ONE) \ 595 | ( (alluser) << 3*TLBINV_ALLUSER) \ 596 | ((allkernel) << 3*TLBINV_ALLKERNEL) \ 597 | ( (all) << 3*TLBINV_ALL)) >> 3*(op)) & 7) 598 599 #define TLBINV_USER_MAP(op) \ 600 TLBINV_MAP(op, TLBINV_ONE, TLBINV_ALLUSER, TLBINV_ALLUSER, \ 601 TLBINV_ALL, TLBINV_ALL) 602 603 #define TLBINV_KERNEL_MAP(op) \ 604 TLBINV_MAP(op, TLBINV_ALLKERNEL, TLBINV_ALL, TLBINV_ALL, \ 605 TLBINV_ALLKERNEL, TLBINV_ALL) 606 607 bool 608 pmap_tlb_shootdown_bystanders(pmap_t pm) 609 { 610 /* 611 * We don't need to deal our own TLB. 612 */ 613 kcpuset_t *pm_active; 614 615 kcpuset_clone(&pm_active, pm->pm_active); 616 kcpuset_atomicly_remove(pm->pm_active, 617 cpu_tlb_info(curcpu())->ti_kcpuset); 618 const bool kernel_p = (pm == pmap_kernel()); 619 bool ipi_sent = false; 620 621 /* 622 * If pm_active gets more bits set, then it's after all our changes 623 * have been made so they will already be cognizant of them. 624 */ 625 626 for (size_t i = 0; !kcpuset_iszero(pm_active); i++) { 627 KASSERT(i < pmap_ntlbs); 628 struct pmap_tlb_info * const ti = pmap_tlbs[i]; 629 KASSERT(tlbinfo_index(ti) == i); 630 /* 631 * Skip this TLB if there are no active mappings for it. 632 */ 633 if (!kcpuset_intersecting_p(pm_active, ti->ti_kcpuset)) 634 continue; 635 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 636 kcpuset_remove(pm_active, ti->ti_kcpuset); 637 TLBINFO_LOCK(ti); 638 if (pmap_tlb_intersecting_onproc_p(pm, ti)) { 639 cpuid_t j = kcpuset_ffs_intersecting(pm->pm_onproc, 640 ti->ti_kcpuset); 641 if (kernel_p) { 642 ti->ti_tlbinvop = 643 TLBINV_KERNEL_MAP(ti->ti_tlbinvop); 644 ti->ti_victim = NULL; 645 } else { 646 KASSERT(pai->pai_asid); 647 if (__predict_false(ti->ti_victim == pm)) { 648 KASSERT(ti->ti_tlbinvop == TLBINV_ONE); 649 /* 650 * We still need to invalidate this one 651 * ASID so there's nothing to change. 652 */ 653 } else { 654 ti->ti_tlbinvop = 655 TLBINV_USER_MAP(ti->ti_tlbinvop); 656 if (ti->ti_tlbinvop == TLBINV_ONE) 657 ti->ti_victim = pm; 658 else 659 ti->ti_victim = NULL; 660 } 661 } 662 TLBINFO_UNLOCK(ti); 663 /* 664 * Now we can send out the shootdown IPIs to a CPU 665 * that shares this TLB and is currently using this 666 * pmap. That CPU will process the IPI and do the 667 * all the work. Any other CPUs sharing that TLB 668 * will take advantage of that work. pm_onproc might 669 * change now that we have released the lock but we 670 * can tolerate spurious shootdowns. 671 */ 672 cpu_send_ipi(cpu_lookup(j), IPI_SHOOTDOWN); 673 ipi_sent = true; 674 continue; 675 } 676 if (!pmap_tlb_intersecting_active_p(pm, ti)) { 677 /* 678 * If this pmap has an ASID assigned but it's not 679 * currently running, nuke its ASID. Next time the 680 * pmap is activated, it will allocate a new ASID. 681 * And best of all, we avoid an IPI. 682 */ 683 KASSERT(!kernel_p); 684 pmap_pai_reset(ti, pai, pm); 685 //ti->ti_evcnt_lazy_shots.ev_count++; 686 } 687 TLBINFO_UNLOCK(ti); 688 } 689 690 kcpuset_destroy(pm_active); 691 692 return ipi_sent; 693 } 694 #endif /* MULTIPROCESSOR && PMAP_NEED_TLB_SHOOTDOWN */ 695 696 #ifndef PMAP_TLB_HWPAGEWALKER 697 int 698 pmap_tlb_update_addr(pmap_t pm, vaddr_t va, pt_entry_t pt_entry, u_int flags) 699 { 700 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 701 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 702 int rv = -1; 703 704 KASSERT(kpreempt_disabled()); 705 706 TLBINFO_LOCK(ti); 707 if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) { 708 pmap_tlb_asid_check(); 709 rv = tlb_update_addr(va, pai->pai_asid, pt_entry, 710 (flags & PMAP_TLB_INSERT) != 0); 711 pmap_tlb_asid_check(); 712 } 713 #if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN) 714 pm->pm_shootdown_pending = (flags & PMAP_TLB_NEED_IPI) != 0; 715 #endif 716 TLBINFO_UNLOCK(ti); 717 718 return rv; 719 } 720 #endif /* !PMAP_TLB_HWPAGEWALKER */ 721 722 void 723 pmap_tlb_invalidate_addr(pmap_t pm, vaddr_t va) 724 { 725 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 726 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 727 728 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 729 730 KASSERT(kpreempt_disabled()); 731 732 UVMHIST_LOG(maphist, " (pm=%#x va=%#x) ti=%#x asid=%#x", 733 pm, va, ti, pai->pai_asid); 734 735 TLBINFO_LOCK(ti); 736 if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) { 737 pmap_tlb_asid_check(); 738 UVMHIST_LOG(maphist, " invalidating %#x asid %#x", 739 va, pai->pai_asid, 0, 0); 740 tlb_invalidate_addr(va, pai->pai_asid); 741 pmap_tlb_asid_check(); 742 } 743 #if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN) 744 pm->pm_shootdown_pending = 1; 745 #endif 746 TLBINFO_UNLOCK(ti); 747 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 748 } 749 750 static inline void 751 pmap_tlb_asid_alloc(struct pmap_tlb_info *ti, pmap_t pm, 752 struct pmap_asid_info *pai) 753 { 754 /* 755 * We shouldn't have an ASID assigned, and thusly must not be onproc 756 * nor active. 757 */ 758 KASSERT(pm != pmap_kernel()); 759 KASSERT(pai->pai_asid == 0); 760 KASSERT(pai->pai_link.le_prev == NULL); 761 #if defined(MULTIPROCESSOR) 762 KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti)); 763 KASSERT(!pmap_tlb_intersecting_active_p(pm, ti)); 764 #endif 765 KASSERT(ti->ti_asids_free > 0); 766 KASSERT(ti->ti_asid_hint > KERNEL_PID); 767 768 /* 769 * If the last ASID allocated was the maximum ASID, then the 770 * hint will be out of range. Reset the hint to first 771 * available ASID. 772 */ 773 if (PMAP_TLB_FLUSH_ASID_ON_RESET 774 && ti->ti_asid_hint > ti->ti_asid_max) { 775 ti->ti_asid_hint = KERNEL_PID + 1; 776 } 777 KASSERTMSG(ti->ti_asid_hint <= ti->ti_asid_max, "hint %u", 778 ti->ti_asid_hint); 779 780 /* 781 * Let's see if the hinted ASID is free. If not search for 782 * a new one. 783 */ 784 if (__predict_true(TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint))) { 785 const size_t nbpw __diagused = 8*sizeof(ti->ti_asid_bitmap[0]); 786 size_t i; 787 u_long bits; 788 for (i = 0; (bits = ~ti->ti_asid_bitmap[i]) == 0; i++) { 789 KASSERT(i < __arraycount(ti->ti_asid_bitmap) - 1); 790 } 791 /* 792 * ffs wants to find the first bit set while we want 793 * to find the first bit cleared. 794 */ 795 const u_int n = __builtin_ffsl(bits) - 1; 796 KASSERTMSG((bits << (nbpw - (n+1))) == (1ul << (nbpw-1)), 797 "n %u bits %#lx", n, bits); 798 KASSERT(n < nbpw); 799 ti->ti_asid_hint = n + i * nbpw; 800 } 801 802 KASSERT(ti->ti_asid_hint > KERNEL_PID); 803 KASSERT(ti->ti_asid_hint <= ti->ti_asid_max); 804 KASSERTMSG(PMAP_TLB_FLUSH_ASID_ON_RESET 805 || TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint - 1), 806 "hint %u bitmap %p", ti->ti_asid_hint, ti->ti_asid_bitmap); 807 KASSERTMSG(!TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint), 808 "hint %u bitmap %p", ti->ti_asid_hint, ti->ti_asid_bitmap); 809 810 /* 811 * The hint contains our next ASID so take it and advance the hint. 812 * Mark it as used and insert the pai into the list of active asids. 813 * There is also one less asid free in this TLB. 814 */ 815 KASSERT(ti->ti_asid_hint > KERNEL_PID); 816 pai->pai_asid = ti->ti_asid_hint++; 817 #ifdef MULTIPROCESSOR 818 if (PMAP_TLB_FLUSH_ASID_ON_RESET) { 819 /* 820 * Clean the new ASID from the TLB. 821 */ 822 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid); 823 } 824 #endif 825 TLBINFO_ASID_MARK_USED(ti, pai->pai_asid); 826 LIST_INSERT_HEAD(&ti->ti_pais, pai, pai_link); 827 ti->ti_asids_free--; 828 829 #if defined(MULTIPROCESSOR) 830 /* 831 * Mark that we now have an active ASID for all CPUs sharing this TLB. 832 * The bits in pm_active belonging to this TLB can only be changed 833 * while this TLBs lock is held. 834 */ 835 #if PMAP_TLB_MAX == 1 836 kcpuset_copy(pm->pm_active, kcpuset_running); 837 #else 838 kcpuset_atomicly_merge(pm->pm_active, ti->ti_kcpuset); 839 #endif 840 #endif 841 } 842 843 /* 844 * Acquire a TLB address space tag (called ASID or TLBPID) and return it. 845 * ASID might have already been previously acquired. 846 */ 847 void 848 pmap_tlb_asid_acquire(pmap_t pm, struct lwp *l) 849 { 850 struct cpu_info * const ci = l->l_cpu; 851 struct pmap_tlb_info * const ti = cpu_tlb_info(ci); 852 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 853 854 KASSERT(kpreempt_disabled()); 855 856 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 857 858 /* 859 * Kernels use a fixed ASID and thus doesn't need to acquire one. 860 */ 861 if (pm == pmap_kernel()) { 862 UVMHIST_LOG(maphist, " <-- done (kernel)", 0, 0, 0, 0); 863 return; 864 } 865 866 UVMHIST_LOG(maphist, " (pm=%#x, l=%#x, ti=%#x)", pm, l, ti, 0); 867 TLBINFO_LOCK(ti); 868 KASSERT(pai->pai_asid <= KERNEL_PID || pai->pai_link.le_prev != NULL); 869 KASSERT(pai->pai_asid > KERNEL_PID || pai->pai_link.le_prev == NULL); 870 pmap_pai_check(ti); 871 if (__predict_false(!PMAP_PAI_ASIDVALID_P(pai, ti))) { 872 /* 873 * If we've run out ASIDs, reinitialize the ASID space. 874 */ 875 if (__predict_false(tlbinfo_noasids_p(ti))) { 876 KASSERT(l == curlwp); 877 UVMHIST_LOG(maphist, " asid reinit", 0, 0, 0, 0); 878 pmap_tlb_asid_reinitialize(ti, TLBINV_NOBODY); 879 KASSERT(!tlbinfo_noasids_p(ti)); 880 } 881 882 /* 883 * Get an ASID. 884 */ 885 pmap_tlb_asid_alloc(ti, pm, pai); 886 UVMHIST_LOG(maphist, "allocated asid %#x", pai->pai_asid, 0, 0, 0); 887 } 888 889 if (l == curlwp) { 890 #if defined(MULTIPROCESSOR) 891 /* 892 * The bits in pm_onproc belonging to this TLB can only 893 * be changed while this TLBs lock is held unless atomic 894 * operations are used. 895 */ 896 KASSERT(pm != pmap_kernel()); 897 kcpuset_atomic_set(pm->pm_onproc, cpu_index(ci)); 898 #endif 899 ci->ci_pmap_asid_cur = pai->pai_asid; 900 UVMHIST_LOG(maphist, "setting asid to %#x", pai->pai_asid, 0, 0, 0); 901 tlb_set_asid(pai->pai_asid); 902 pmap_tlb_asid_check(); 903 } else { 904 printf("%s: l (%p) != curlwp %p\n", __func__, l, curlwp); 905 } 906 TLBINFO_UNLOCK(ti); 907 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 908 } 909 910 void 911 pmap_tlb_asid_deactivate(pmap_t pm) 912 { 913 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 914 KASSERT(kpreempt_disabled()); 915 #if defined(MULTIPROCESSOR) 916 /* 917 * The kernel pmap is aways onproc and active and must never have 918 * those bits cleared. If pmap_remove_all was called, it has already 919 * deactivated the pmap and thusly onproc will be 0 so there's nothing 920 * to do. 921 */ 922 if (pm != pmap_kernel() && !kcpuset_iszero(pm->pm_onproc)) { 923 struct cpu_info * const ci = curcpu(); 924 KASSERT(!cpu_intr_p()); 925 KASSERTMSG(kcpuset_isset(pm->pm_onproc, cpu_index(ci)), 926 "%s: pmap %p onproc %p doesn't include cpu %d (%p)", 927 __func__, pm, pm->pm_onproc, cpu_index(ci), ci); 928 /* 929 * The bits in pm_onproc that belong to this TLB can 930 * be changed while this TLBs lock is not held as long 931 * as we use atomic ops. 932 */ 933 kcpuset_atomic_clear(pm->pm_onproc, cpu_index(ci)); 934 } 935 #endif 936 curcpu()->ci_pmap_asid_cur = 0; 937 UVMHIST_LOG(maphist, " <-- done (pm=%#x)", pm, 0, 0, 0); 938 tlb_set_asid(0); 939 #if defined(DEBUG) 940 pmap_tlb_asid_check(); 941 #endif 942 } 943 944 void 945 pmap_tlb_asid_release_all(struct pmap *pm) 946 { 947 KASSERT(pm != pmap_kernel()); 948 #if defined(MULTIPROCESSOR) 949 //KASSERT(!kcpuset_iszero(pm->pm_onproc)); // XXX 950 #if PMAP_TLB_MAX > 1 951 struct cpu_info * const ci __diagused = curcpu(); 952 for (u_int i = 0; !kcpuset_iszero(pm->pm_active); i++) { 953 KASSERT(i < pmap_ntlbs); 954 struct pmap_tlb_info * const ti = pmap_tlbs[i]; 955 #else 956 struct pmap_tlb_info * const ti = &pmap_tlb0_info; 957 #endif 958 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 959 TLBINFO_LOCK(ti); 960 if (PMAP_PAI_ASIDVALID_P(pai, ti)) { 961 /* 962 * If this pmap isn't onproc on any of the cpus 963 * belonging to this tlb domain, we can just reset 964 * the ASID and be done. 965 */ 966 if (!pmap_tlb_intersecting_onproc_p(pm, ti)) { 967 KASSERT(ti->ti_victim != pm); 968 pmap_pai_reset(ti, pai, pm); 969 #if PMAP_TLB_MAX == 1 970 } else { 971 KASSERT(cpu_tlb_info(ci) == ti); 972 tlb_invalidate_asids(pai->pai_asid, 973 pai->pai_asid); 974 #else 975 } else if (cpu_tlb_info(ci) == ti) { 976 tlb_invalidate_asids(pai->pai_asid, 977 pai->pai_asid); 978 } else { 979 pm->pm_shootdown_needed = 1; 980 #endif 981 } 982 } 983 TLBINFO_UNLOCK(ti); 984 #if PMAP_TLB_MAX > 1 985 } 986 #endif 987 #else 988 /* 989 * Handle the case of an UP kernel which only has, at most, one ASID. 990 * If the pmap has an ASID allocated, free it. 991 */ 992 struct pmap_tlb_info * const ti = &pmap_tlb0_info; 993 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 994 TLBINFO_LOCK(ti); 995 if (pai->pai_asid > KERNEL_PID) { 996 if (curcpu()->ci_pmap_asid_cur == pai->pai_asid) { 997 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid); 998 } else { 999 pmap_pai_reset(ti, pai, pm); 1000 } 1001 } 1002 TLBINFO_UNLOCK(ti); 1003 #endif /* MULTIPROCESSOR */ 1004 } 1005 1006 void 1007 pmap_tlb_asid_check(void) 1008 { 1009 #ifdef DEBUG 1010 kpreempt_disable(); 1011 const tlb_asid_t asid __debugused = tlb_get_asid(); 1012 KDASSERTMSG(asid == curcpu()->ci_pmap_asid_cur, 1013 "%s: asid (%#x) != current asid (%#x)", 1014 __func__, asid, curcpu()->ci_pmap_asid_cur); 1015 kpreempt_enable(); 1016 #endif 1017 } 1018 1019 #ifdef DEBUG 1020 void 1021 pmap_tlb_check(pmap_t pm, bool (*func)(void *, vaddr_t, tlb_asid_t, pt_entry_t)) 1022 { 1023 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 1024 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 1025 TLBINFO_LOCK(ti); 1026 if (pm == pmap_kernel() || pai->pai_asid > KERNEL_PID) 1027 tlb_walk(pm, func); 1028 TLBINFO_UNLOCK(ti); 1029 } 1030 #endif /* DEBUG */ 1031