1 /* $NetBSD: e500_tlb.c,v 1.18 2016/07/11 16:06:52 matt Exp $ */ 2 /*- 3 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects 8 * Agency and which was developed by Matt Thomas of 3am Software Foundry. 9 * 10 * This material is based upon work supported by the Defense Advanced Research 11 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under 12 * Contract No. N66001-09-C-2073. 13 * Approved for Public Release, Distribution Unlimited 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 26 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34 * POSSIBILITY OF SUCH DAMAGE. 35 */ 36 37 #include "opt_ppcparam.h" 38 39 #define __PMAP_PRIVATE 40 41 #include <sys/cdefs.h> 42 43 __KERNEL_RCSID(0, "$NetBSD: e500_tlb.c,v 1.18 2016/07/11 16:06:52 matt Exp $"); 44 45 #include <sys/param.h> 46 47 #include <uvm/uvm_extern.h> 48 49 #include <powerpc/spr.h> 50 #include <powerpc/booke/spr.h> 51 #include <powerpc/booke/cpuvar.h> 52 #include <powerpc/booke/e500reg.h> 53 #include <powerpc/booke/e500var.h> 54 #include <powerpc/booke/pmap.h> 55 56 struct e500_tlb { 57 vaddr_t tlb_va; 58 uint32_t tlb_pte; 59 uint32_t tlb_asid; 60 vsize_t tlb_size; 61 }; 62 63 struct e500_hwtlb { 64 uint32_t hwtlb_mas0; 65 uint32_t hwtlb_mas1; 66 uint32_t hwtlb_mas2; 67 uint32_t hwtlb_mas3; 68 }; 69 70 struct e500_xtlb { 71 struct e500_tlb e_tlb; 72 struct e500_hwtlb e_hwtlb; 73 u_long e_refcnt; 74 }; 75 76 static struct e500_tlb1 { 77 uint32_t tlb1_maxsize; 78 uint32_t tlb1_minsize; 79 u_int tlb1_numentries; 80 u_int tlb1_numfree; 81 u_int tlb1_freelist[32]; 82 struct e500_xtlb tlb1_entries[32]; 83 } e500_tlb1; 84 85 static inline register_t mftlb0cfg(void) __pure; 86 static inline register_t mftlb1cfg(void) __pure; 87 88 static inline register_t 89 mftlb0cfg(void) 90 { 91 register_t tlb0cfg; 92 __asm("mfspr %0, %1" : "=r"(tlb0cfg) : "n"(SPR_TLB0CFG)); 93 return tlb0cfg; 94 } 95 96 static inline register_t 97 mftlb1cfg(void) 98 { 99 register_t tlb1cfg; 100 __asm("mfspr %0, %1" : "=r"(tlb1cfg) : "n"(SPR_TLB1CFG)); 101 return tlb1cfg; 102 } 103 104 static struct e500_tlb 105 hwtlb_to_tlb(const struct e500_hwtlb hwtlb) 106 { 107 struct e500_tlb tlb; 108 register_t prot_mask; 109 u_int prot_shift; 110 111 tlb.tlb_va = MAS2_EPN & hwtlb.hwtlb_mas2; 112 tlb.tlb_size = 1024 << (2 * MASX_TSIZE_GET(hwtlb.hwtlb_mas1)); 113 tlb.tlb_asid = MASX_TID_GET(hwtlb.hwtlb_mas1); 114 tlb.tlb_pte = (hwtlb.hwtlb_mas2 & MAS2_WIMGE) 115 | (hwtlb.hwtlb_mas3 & MAS3_RPN); 116 if (hwtlb.hwtlb_mas1 & MAS1_TS) { 117 prot_mask = MAS3_UX|MAS3_UW|MAS3_UR; 118 prot_shift = PTE_RWX_SHIFT - 1; 119 } else { 120 prot_mask = MAS3_SX|MAS3_SW|MAS3_SR; 121 prot_shift = PTE_RWX_SHIFT; 122 } 123 tlb.tlb_pte |= (prot_mask & hwtlb.hwtlb_mas3) << prot_shift; 124 return tlb; 125 } 126 127 static inline struct e500_hwtlb 128 hwtlb_read(uint32_t mas0, u_int slot) 129 { 130 struct e500_hwtlb hwtlb; 131 register_t tlbcfg; 132 133 if (__predict_true(mas0 == MAS0_TLBSEL_TLB0)) { 134 tlbcfg = mftlb0cfg(); 135 } else if (mas0 == MAS0_TLBSEL_TLB1) { 136 tlbcfg = mftlb1cfg(); 137 } else { 138 panic("%s:%d: unexpected MAS0 %#" PRIx32, 139 __func__, __LINE__, mas0); 140 } 141 142 /* 143 * ESEL is the way we want to look up. 144 * If tlbassoc is the same as tlbentries (like in TLB1) then the TLB is 145 * fully associative, the entire slot is placed into ESEL. If tlbassoc 146 * is less than the number of tlb entries, the slot is split in two 147 * fields. Since the TLB is M rows by N ways, the lowers bits are for 148 * row (MAS2[EPN]) and the upper for the way (MAS1[ESEL]). 149 */ 150 const u_int tlbassoc = TLBCFG_ASSOC(tlbcfg); 151 const u_int tlbentries = TLBCFG_NENTRY(tlbcfg); 152 const u_int esel_shift = 153 __builtin_clz(tlbassoc) - __builtin_clz(tlbentries); 154 155 /* 156 * Disable interrupts since we don't want anyone else mucking with 157 * the MMU Assist registers 158 */ 159 const register_t msr = wrtee(0); 160 const register_t saved_mas0 = mfspr(SPR_MAS0); 161 mtspr(SPR_MAS0, mas0 | MAS0_ESEL_MAKE(slot >> esel_shift)); 162 163 if (__predict_true(tlbassoc > tlbentries)) 164 mtspr(SPR_MAS2, slot << PAGE_SHIFT); 165 166 /* 167 * Now select the entry and grab its contents. 168 */ 169 __asm volatile("tlbre"); 170 171 hwtlb.hwtlb_mas0 = mfspr(SPR_MAS0); 172 hwtlb.hwtlb_mas1 = mfspr(SPR_MAS1); 173 hwtlb.hwtlb_mas2 = mfspr(SPR_MAS2); 174 hwtlb.hwtlb_mas3 = mfspr(SPR_MAS3); 175 176 mtspr(SPR_MAS0, saved_mas0); 177 wrtee(msr); /* restore interrupts */ 178 179 return hwtlb; 180 } 181 182 static inline void 183 hwtlb_write(const struct e500_hwtlb hwtlb, bool needs_sync) 184 { 185 const register_t msr = wrtee(0); 186 const uint32_t saved_mas0 = mfspr(SPR_MAS0); 187 188 /* 189 * Need to always write MAS0 and MAS1 190 */ 191 mtspr(SPR_MAS0, hwtlb.hwtlb_mas0); 192 mtspr(SPR_MAS1, hwtlb.hwtlb_mas1); 193 194 /* 195 * Only write the VPN/WIMGE if this is in TLB0 or if a valid mapping. 196 */ 197 if ((hwtlb.hwtlb_mas0 & MAS0_TLBSEL) == MAS0_TLBSEL_TLB0 198 || (hwtlb.hwtlb_mas1 & MAS1_V)) { 199 mtspr(SPR_MAS2, hwtlb.hwtlb_mas2); 200 } 201 /* 202 * Only need to write the RPN/prot if we are dealing with a valid 203 * mapping. 204 */ 205 if (hwtlb.hwtlb_mas1 & MAS1_V) { 206 mtspr(SPR_MAS3, hwtlb.hwtlb_mas3); 207 //mtspr(SPR_MAS7, 0); 208 } 209 210 #if 0 211 printf("%s->[%x,%x,%x,%x]\n", 212 __func__, 213 hwtlb.hwtlb_mas0, hwtlb.hwtlb_mas1, 214 hwtlb.hwtlb_mas2, hwtlb.hwtlb_mas3); 215 #endif 216 __asm volatile("tlbwe"); 217 if (needs_sync) { 218 __asm volatile("tlbsync\n\tisync\n\tsync"); 219 } 220 221 mtspr(SPR_MAS0, saved_mas0); 222 wrtee(msr); 223 } 224 225 static struct e500_hwtlb 226 tlb_to_hwtlb(const struct e500_tlb tlb) 227 { 228 struct e500_hwtlb hwtlb; 229 230 KASSERT(trunc_page(tlb.tlb_va) == tlb.tlb_va); 231 KASSERT(tlb.tlb_size != 0); 232 KASSERT((tlb.tlb_size & (tlb.tlb_size - 1)) == 0); 233 const uint32_t prot_mask = tlb.tlb_pte & PTE_RWX_MASK; 234 if (__predict_true(tlb.tlb_size == PAGE_SIZE)) { 235 hwtlb.hwtlb_mas0 = 0; 236 hwtlb.hwtlb_mas1 = MAS1_V | MASX_TSIZE_MAKE(1); 237 /* 238 * A non-zero ASID means this is a user page so mark it as 239 * being in the user's address space. 240 */ 241 if (tlb.tlb_asid) { 242 hwtlb.hwtlb_mas1 |= MAS1_TS 243 | MASX_TID_MAKE(tlb.tlb_asid); 244 hwtlb.hwtlb_mas3 = (prot_mask >> (PTE_RWX_SHIFT - 1)) 245 | ((prot_mask & ~PTE_xX) >> PTE_RWX_SHIFT); 246 KASSERT(prot_mask & PTE_xR); 247 KASSERT(hwtlb.hwtlb_mas3 & MAS3_UR); 248 CTASSERT(MAS3_UR == (PTE_xR >> (PTE_RWX_SHIFT - 1))); 249 CTASSERT(MAS3_SR == (PTE_xR >> PTE_RWX_SHIFT)); 250 } else { 251 hwtlb.hwtlb_mas3 = prot_mask >> PTE_RWX_SHIFT; 252 } 253 if (tlb.tlb_pte & PTE_UNMODIFIED) 254 hwtlb.hwtlb_mas3 &= ~(MAS3_UW|MAS3_SW); 255 if (tlb.tlb_pte & PTE_UNSYNCED) 256 hwtlb.hwtlb_mas3 &= ~(MAS3_UX|MAS3_SX); 257 } else { 258 KASSERT(tlb.tlb_asid == 0); 259 KASSERT((tlb.tlb_size & 0xaaaaa7ff) == 0); 260 u_int cntlz = __builtin_clz(tlb.tlb_size); 261 KASSERT(cntlz & 1); 262 KASSERT(cntlz <= 19); 263 hwtlb.hwtlb_mas0 = MAS0_TLBSEL_TLB1; 264 /* 265 * TSIZE is defined (4^TSIZE) Kbytes except a TSIZE of 0 is not 266 * allowed. So 1K would be 0x00000400 giving 21 leading zero 267 * bits. Subtracting the leading number of zero bits from 21 268 * and dividing by 2 gives us the number that the MMU wants. 269 */ 270 hwtlb.hwtlb_mas1 = MASX_TSIZE_MAKE(((31 - 10) - cntlz) / 2) 271 | MAS1_IPROT | MAS1_V; 272 hwtlb.hwtlb_mas3 = prot_mask >> PTE_RWX_SHIFT; 273 } 274 /* We are done with MAS1, on to MAS2 ... */ 275 hwtlb.hwtlb_mas2 = tlb.tlb_va | (tlb.tlb_pte & PTE_WIMGE_MASK); 276 hwtlb.hwtlb_mas3 |= tlb.tlb_pte & PTE_RPN_MASK; 277 278 return hwtlb; 279 } 280 281 void * 282 e500_tlb1_fetch(size_t slot) 283 { 284 struct e500_tlb1 * const tlb1 = &e500_tlb1; 285 286 return &tlb1->tlb1_entries[slot].e_hwtlb; 287 } 288 289 void 290 e500_tlb1_sync(void) 291 { 292 struct e500_tlb1 * const tlb1 = &e500_tlb1; 293 for (u_int slot = 1; slot < tlb1->tlb1_numentries; slot++) { 294 const struct e500_hwtlb * const new_hwtlb = 295 &tlb1->tlb1_entries[slot].e_hwtlb; 296 const struct e500_hwtlb old_hwtlb = 297 hwtlb_read(MAS0_TLBSEL_TLB1, slot); 298 #define CHANGED(n,o,f) ((n)->f != (o).f) 299 bool mas1_changed_p = CHANGED(new_hwtlb, old_hwtlb, hwtlb_mas1); 300 bool mas2_changed_p = CHANGED(new_hwtlb, old_hwtlb, hwtlb_mas2); 301 bool mas3_changed_p = CHANGED(new_hwtlb, old_hwtlb, hwtlb_mas3); 302 #undef CHANGED 303 bool new_valid_p = (new_hwtlb->hwtlb_mas1 & MAS1_V) != 0; 304 bool old_valid_p = (old_hwtlb.hwtlb_mas1 & MAS1_V) != 0; 305 if ((new_valid_p || old_valid_p) 306 && (mas1_changed_p 307 || (new_valid_p 308 && (mas2_changed_p || mas3_changed_p)))) 309 hwtlb_write(*new_hwtlb, true); 310 } 311 } 312 313 static int 314 e500_alloc_tlb1_entry(void) 315 { 316 struct e500_tlb1 * const tlb1 = &e500_tlb1; 317 318 if (tlb1->tlb1_numfree == 0) 319 return -1; 320 const u_int slot = tlb1->tlb1_freelist[--tlb1->tlb1_numfree]; 321 KASSERT((tlb1->tlb1_entries[slot].e_hwtlb.hwtlb_mas1 & MAS1_V) == 0); 322 tlb1->tlb1_entries[slot].e_hwtlb.hwtlb_mas0 = 323 MAS0_TLBSEL_TLB1 | __SHIFTIN(slot, MAS0_ESEL); 324 return (int)slot; 325 } 326 327 static void 328 e500_free_tlb1_entry(struct e500_xtlb *xtlb, u_int slot, bool needs_sync) 329 { 330 struct e500_tlb1 * const tlb1 = &e500_tlb1; 331 KASSERT(slot < tlb1->tlb1_numentries); 332 KASSERT(&tlb1->tlb1_entries[slot] == xtlb); 333 334 KASSERT(xtlb->e_hwtlb.hwtlb_mas0 == (MAS0_TLBSEL_TLB1|__SHIFTIN(slot, MAS0_ESEL))); 335 xtlb->e_hwtlb.hwtlb_mas1 &= ~(MAS1_V|MAS1_IPROT); 336 hwtlb_write(xtlb->e_hwtlb, needs_sync); 337 338 const register_t msr = wrtee(0); 339 tlb1->tlb1_freelist[tlb1->tlb1_numfree++] = slot; 340 wrtee(msr); 341 } 342 343 static tlb_asid_t 344 e500_tlb_get_asid(void) 345 { 346 return mfspr(SPR_PID0); 347 } 348 349 static void 350 e500_tlb_set_asid(tlb_asid_t asid) 351 { 352 mtspr(SPR_PID0, asid); 353 } 354 355 static void 356 e500_tlb_invalidate_all(void) 357 { 358 /* 359 * This does a flash invalidate of all entries in TLB0. 360 * We don't touch TLB1 since we don't expect those to be volatile. 361 */ 362 #if 1 363 __asm volatile("tlbivax\t0, %0" :: "b"(4)); /* INV_ALL */ 364 __asm volatile("tlbsync\n\tisync\n\tsync"); 365 #else 366 mtspr(SPR_MMUCSR0, MMUCSR0_TLB0_FI); 367 while (mfspr(SPR_MMUCSR0) != 0) 368 ; 369 #endif 370 } 371 372 static void 373 e500_tlb_invalidate_globals(void) 374 { 375 #if defined(MULTIPROCESSOR) 376 e500_tlb_invalidate_all(); 377 #else /* !MULTIPROCESSOR */ 378 const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg()); 379 const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg()); 380 const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT; 381 const vaddr_t kstack_lo = (uintptr_t)curlwp->l_addr; 382 const vaddr_t kstack_hi = kstack_lo + USPACE - 1; 383 const vaddr_t epn_kstack_lo = kstack_lo & (max_epn - 1); 384 const vaddr_t epn_kstack_hi = kstack_hi & (max_epn - 1); 385 386 const register_t msr = wrtee(0); 387 for (size_t assoc = 0; assoc < tlbassoc; assoc++) { 388 mtspr(SPR_MAS0, MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0); 389 for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) { 390 mtspr(SPR_MAS2, epn); 391 __asm volatile("tlbre"); 392 uint32_t mas1 = mfspr(SPR_MAS1); 393 394 /* 395 * Make sure this is a valid kernel entry first. 396 */ 397 if ((mas1 & (MAS1_V|MAS1_TID|MAS1_TS)) != MAS1_V) 398 continue; 399 400 /* 401 * We have a valid kernel TLB entry. But if it matches 402 * the stack we are currently running on, it would 403 * unwise to invalidate it. First see if the epn 404 * overlaps the stack. If it does then get the 405 * VA and see if it really is part of the stack. 406 */ 407 if (epn_kstack_lo < epn_kstack_hi 408 ? (epn_kstack_lo <= epn && epn <= epn_kstack_hi) 409 : (epn <= epn_kstack_hi || epn_kstack_lo <= epn)) { 410 const uint32_t mas2_epn = 411 mfspr(SPR_MAS2) & MAS2_EPN; 412 if (kstack_lo <= mas2_epn 413 && mas2_epn <= kstack_hi) 414 continue; 415 } 416 mtspr(SPR_MAS1, mas1 ^ MAS1_V); 417 __asm volatile("tlbwe"); 418 } 419 } 420 __asm volatile("isync\n\tsync"); 421 wrtee(msr); 422 #endif /* MULTIPROCESSOR */ 423 } 424 425 static void 426 e500_tlb_invalidate_asids(tlb_asid_t asid_lo, tlb_asid_t asid_hi) 427 { 428 #if defined(MULTIPROCESSOR) 429 e500_tlb_invalidate_all(); 430 #else /* !MULTIPROCESSOR */ 431 const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg()); 432 const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg()); 433 const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT; 434 435 asid_lo = __SHIFTIN(asid_lo, MAS1_TID); 436 asid_hi = __SHIFTIN(asid_hi, MAS1_TID); 437 438 const register_t msr = wrtee(0); 439 for (size_t assoc = 0; assoc < tlbassoc; assoc++) { 440 mtspr(SPR_MAS0, MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0); 441 for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) { 442 mtspr(SPR_MAS2, epn); 443 __asm volatile("tlbre"); 444 const uint32_t mas1 = mfspr(SPR_MAS1); 445 /* 446 * If this is a valid entry for AS space 1 and 447 * its asid matches the constraints of the caller, 448 * clear its valid bit. 449 */ 450 if ((mas1 & (MAS1_V|MAS1_TS)) == (MAS1_V|MAS1_TS) 451 && asid_lo <= (mas1 & MAS1_TID) 452 && (mas1 & MAS1_TID) <= asid_hi) { 453 mtspr(SPR_MAS1, mas1 ^ MAS1_V); 454 #if 0 455 printf("%s[%zu,%zu]->[%x]\n", 456 __func__, assoc, epn, mas1); 457 #endif 458 __asm volatile("tlbwe"); 459 } 460 } 461 } 462 __asm volatile("isync\n\tsync"); 463 wrtee(msr); 464 #endif /* MULTIPROCESSOR */ 465 } 466 467 static u_int 468 e500_tlb_record_asids(u_long *bitmap, tlb_asid_t asid_max) 469 { 470 const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg()); 471 const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg()); 472 const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT; 473 const size_t nbits = 8 * sizeof(bitmap[0]); 474 u_int found = 0; 475 476 const register_t msr = wrtee(0); 477 for (size_t assoc = 0; assoc < tlbassoc; assoc++) { 478 mtspr(SPR_MAS0, MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0); 479 for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) { 480 mtspr(SPR_MAS2, epn); 481 __asm volatile("tlbre"); 482 const uint32_t mas1 = mfspr(SPR_MAS1); 483 /* 484 * If this is a valid entry for AS space 1 and 485 * its asid matches the constraints of the caller, 486 * clear its valid bit. 487 */ 488 if ((mas1 & (MAS1_V|MAS1_TS)) == (MAS1_V|MAS1_TS)) { 489 const uint32_t asid = MASX_TID_GET(mas1); 490 const u_int i = asid / nbits; 491 const u_long mask = 1UL << (asid & (nbits - 1)); 492 if ((bitmap[i] & mask) == 0) { 493 bitmap[i] |= mask; 494 found++; 495 } 496 } 497 } 498 } 499 wrtee(msr); 500 501 return found; 502 } 503 504 static void 505 e500_tlb_invalidate_addr(vaddr_t va, tlb_asid_t asid) 506 { 507 KASSERT((va & PAGE_MASK) == 0); 508 /* 509 * Bits 60 & 61 have meaning 510 */ 511 if (asid == KERNEL_PID) { 512 /* 513 * For data accesses, the context-synchronizing instruction 514 * before tlbwe or tlbivax ensures that all memory accesses 515 * due to preceding instructions have completed to a point 516 * at which they have reported all exceptions they will cause. 517 */ 518 __asm volatile("isync"); 519 } 520 __asm volatile("tlbivax\t0, %0" :: "b"(va)); 521 __asm volatile("tlbsync"); 522 __asm volatile("tlbsync"); /* Why? */ 523 if (asid == KERNEL_PID) { 524 /* 525 * The context-synchronizing instruction after tlbwe or tlbivax 526 * ensures that subsequent accesses (data and instruction) use 527 * the updated value in any TLB entries affected. 528 */ 529 __asm volatile("isync\n\tsync"); 530 } 531 } 532 533 static bool 534 e500_tlb_update_addr(vaddr_t va, tlb_asid_t asid, pt_entry_t pte, bool insert) 535 { 536 #if defined(MULTIPROCESSOR) 537 e500_tlb_invalidate_addr(va, asid); 538 return true; 539 #else /* !MULTIPROCESSOR */ 540 struct e500_hwtlb hwtlb = tlb_to_hwtlb( 541 (struct e500_tlb){ .tlb_va = va, .tlb_asid = asid, 542 .tlb_size = PAGE_SIZE, .tlb_pte = pte,}); 543 544 register_t msr = wrtee(0); 545 mtspr(SPR_MAS6, asid ? __SHIFTIN(asid, MAS6_SPID0) | MAS6_SAS : 0); 546 __asm volatile("tlbsx 0, %0" :: "b"(va)); 547 register_t mas1 = mfspr(SPR_MAS1); 548 if ((mas1 & MAS1_V) == 0) { 549 if (!insert) { 550 wrtee(msr); 551 #if 0 552 printf("%s(%#lx,%#x,%#x,%x)<no update>\n", 553 __func__, va, asid, pte, insert); 554 #endif 555 return false; 556 } 557 mas1 = hwtlb.hwtlb_mas1 | MAS1_V; 558 mtspr(SPR_MAS1, mas1); 559 } 560 mtspr(SPR_MAS2, hwtlb.hwtlb_mas2); 561 mtspr(SPR_MAS3, hwtlb.hwtlb_mas3); 562 //mtspr(SPR_MAS7, 0); 563 __asm volatile("tlbwe"); 564 if (asid == KERNEL_PID) 565 __asm volatile("isync\n\tsync"); 566 wrtee(msr); 567 #if 0 568 if (asid) 569 printf("%s(%#lx,%#x,%#x,%x)->[%x,%x,%x]\n", 570 __func__, va, asid, pte, insert, 571 hwtlb.hwtlb_mas1, hwtlb.hwtlb_mas2, hwtlb.hwtlb_mas3); 572 #endif 573 return (mas1 & MAS1_V) != 0; 574 #endif /* MULTIPROCESSOR */ 575 } 576 577 static void 578 e500_tlb_write_entry(size_t index, const struct tlbmask *tlb) 579 { 580 } 581 582 static void 583 e500_tlb_read_entry(size_t index, struct tlbmask *tlb) 584 { 585 } 586 587 static void 588 e500_tlb_dump(void (*pr)(const char *, ...)) 589 { 590 const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg()); 591 const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg()); 592 const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT; 593 const uint32_t saved_mas0 = mfspr(SPR_MAS0); 594 size_t valid = 0; 595 596 if (pr == NULL) 597 pr = printf; 598 599 const register_t msr = wrtee(0); 600 for (size_t assoc = 0; assoc < tlbassoc; assoc++) { 601 struct e500_hwtlb hwtlb; 602 hwtlb.hwtlb_mas0 = MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0; 603 mtspr(SPR_MAS0, hwtlb.hwtlb_mas0); 604 for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) { 605 mtspr(SPR_MAS2, epn); 606 __asm volatile("tlbre"); 607 hwtlb.hwtlb_mas1 = mfspr(SPR_MAS1); 608 /* 609 * If this is a valid entry for AS space 1 and 610 * its asid matches the constraints of the caller, 611 * clear its valid bit. 612 */ 613 if (hwtlb.hwtlb_mas1 & MAS1_V) { 614 hwtlb.hwtlb_mas2 = mfspr(SPR_MAS2); 615 hwtlb.hwtlb_mas3 = mfspr(SPR_MAS3); 616 struct e500_tlb tlb = hwtlb_to_tlb(hwtlb); 617 (*pr)("[%zu,%zu]->[%x,%x,%x]", 618 assoc, atop(epn), 619 hwtlb.hwtlb_mas1, 620 hwtlb.hwtlb_mas2, 621 hwtlb.hwtlb_mas3); 622 (*pr)(": VA=%#lx size=4KB asid=%u pte=%x", 623 tlb.tlb_va, tlb.tlb_asid, tlb.tlb_pte); 624 (*pr)(" (RPN=%#x,%s%s%s%s%s,%s%s%s%s%s)\n", 625 tlb.tlb_pte & PTE_RPN_MASK, 626 tlb.tlb_pte & PTE_xR ? "R" : "", 627 tlb.tlb_pte & PTE_xW ? "W" : "", 628 tlb.tlb_pte & PTE_UNMODIFIED ? "*" : "", 629 tlb.tlb_pte & PTE_xX ? "X" : "", 630 tlb.tlb_pte & PTE_UNSYNCED ? "*" : "", 631 tlb.tlb_pte & PTE_W ? "W" : "", 632 tlb.tlb_pte & PTE_I ? "I" : "", 633 tlb.tlb_pte & PTE_M ? "M" : "", 634 tlb.tlb_pte & PTE_G ? "G" : "", 635 tlb.tlb_pte & PTE_E ? "E" : ""); 636 valid++; 637 } 638 } 639 } 640 mtspr(SPR_MAS0, saved_mas0); 641 wrtee(msr); 642 (*pr)("%s: %zu valid entries\n", __func__, valid); 643 } 644 645 static void 646 e500_tlb_walk(void *ctx, bool (*func)(void *, vaddr_t, uint32_t, uint32_t)) 647 { 648 const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg()); 649 const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg()); 650 const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT; 651 const uint32_t saved_mas0 = mfspr(SPR_MAS0); 652 653 const register_t msr = wrtee(0); 654 for (size_t assoc = 0; assoc < tlbassoc; assoc++) { 655 struct e500_hwtlb hwtlb; 656 hwtlb.hwtlb_mas0 = MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0; 657 mtspr(SPR_MAS0, hwtlb.hwtlb_mas0); 658 for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) { 659 mtspr(SPR_MAS2, epn); 660 __asm volatile("tlbre"); 661 hwtlb.hwtlb_mas1 = mfspr(SPR_MAS1); 662 if (hwtlb.hwtlb_mas1 & MAS1_V) { 663 hwtlb.hwtlb_mas2 = mfspr(SPR_MAS2); 664 hwtlb.hwtlb_mas3 = mfspr(SPR_MAS3); 665 struct e500_tlb tlb = hwtlb_to_tlb(hwtlb); 666 if (!(*func)(ctx, tlb.tlb_va, tlb.tlb_asid, 667 tlb.tlb_pte)) 668 break; 669 } 670 } 671 } 672 mtspr(SPR_MAS0, saved_mas0); 673 wrtee(msr); 674 } 675 676 static struct e500_xtlb * 677 e500_tlb_lookup_xtlb_pa(vaddr_t pa, u_int *slotp) 678 { 679 struct e500_tlb1 * const tlb1 = &e500_tlb1; 680 struct e500_xtlb *xtlb = tlb1->tlb1_entries; 681 682 /* 683 * See if we have a TLB entry for the pa. 684 */ 685 for (u_int i = 0; i < tlb1->tlb1_numentries; i++, xtlb++) { 686 psize_t mask = ~(xtlb->e_tlb.tlb_size - 1); 687 if ((xtlb->e_hwtlb.hwtlb_mas1 & MAS1_V) 688 && ((pa ^ xtlb->e_tlb.tlb_pte) & mask) == 0) { 689 if (slotp != NULL) 690 *slotp = i; 691 return xtlb; 692 } 693 } 694 695 return NULL; 696 } 697 698 struct e500_xtlb * 699 e500_tlb_lookup_xtlb(vaddr_t va, u_int *slotp) 700 { 701 struct e500_tlb1 * const tlb1 = &e500_tlb1; 702 struct e500_xtlb *xtlb = tlb1->tlb1_entries; 703 704 /* 705 * See if we have a TLB entry for the va. 706 */ 707 for (u_int i = 0; i < tlb1->tlb1_numentries; i++, xtlb++) { 708 vsize_t mask = ~(xtlb->e_tlb.tlb_size - 1); 709 if ((xtlb->e_hwtlb.hwtlb_mas1 & MAS1_V) 710 && ((va ^ xtlb->e_tlb.tlb_va) & mask) == 0) { 711 if (slotp != NULL) 712 *slotp = i; 713 return xtlb; 714 } 715 } 716 717 return NULL; 718 } 719 720 static struct e500_xtlb * 721 e500_tlb_lookup_xtlb2(vaddr_t va, vsize_t len) 722 { 723 struct e500_tlb1 * const tlb1 = &e500_tlb1; 724 struct e500_xtlb *xtlb = tlb1->tlb1_entries; 725 726 /* 727 * See if we have a TLB entry for the pa. 728 */ 729 for (u_int i = 0; i < tlb1->tlb1_numentries; i++, xtlb++) { 730 vsize_t mask = ~(xtlb->e_tlb.tlb_size - 1); 731 if ((xtlb->e_hwtlb.hwtlb_mas1 & MAS1_V) 732 && ((va ^ xtlb->e_tlb.tlb_va) & mask) == 0 733 && (((va + len - 1) ^ va) & mask) == 0) { 734 return xtlb; 735 } 736 } 737 738 return NULL; 739 } 740 741 static void * 742 e500_tlb_mapiodev(paddr_t pa, psize_t len, bool prefetchable) 743 { 744 struct e500_xtlb * const xtlb = e500_tlb_lookup_xtlb_pa(pa, NULL); 745 746 /* 747 * See if we have a TLB entry for the pa. If completely falls within 748 * mark the reference and return the pa. But only if the tlb entry 749 * is not cacheable. 750 */ 751 if (xtlb 752 && (prefetchable 753 || (xtlb->e_tlb.tlb_pte & PTE_WIG) == (PTE_I|PTE_G))) { 754 xtlb->e_refcnt++; 755 return (void *) (xtlb->e_tlb.tlb_va 756 + pa - (xtlb->e_tlb.tlb_pte & PTE_RPN_MASK)); 757 } 758 return NULL; 759 } 760 761 static void 762 e500_tlb_unmapiodev(vaddr_t va, vsize_t len) 763 { 764 if (va < VM_MIN_KERNEL_ADDRESS || VM_MAX_KERNEL_ADDRESS <= va) { 765 struct e500_xtlb * const xtlb = e500_tlb_lookup_xtlb(va, NULL); 766 if (xtlb) 767 xtlb->e_refcnt--; 768 } 769 } 770 771 static int 772 e500_tlb_ioreserve(vaddr_t va, vsize_t len, pt_entry_t pte) 773 { 774 struct e500_tlb1 * const tlb1 = &e500_tlb1; 775 struct e500_xtlb *xtlb; 776 777 KASSERT(len & 0x55555000); 778 KASSERT((len & ~0x55555000) == 0); 779 KASSERT(len >= PAGE_SIZE); 780 KASSERT((len & (len - 1)) == 0); 781 KASSERT((va & (len - 1)) == 0); 782 KASSERT(((pte & PTE_RPN_MASK) & (len - 1)) == 0); 783 784 if ((xtlb = e500_tlb_lookup_xtlb2(va, len)) != NULL) { 785 psize_t mask __diagused = ~(xtlb->e_tlb.tlb_size - 1); 786 KASSERT(len <= xtlb->e_tlb.tlb_size); 787 KASSERT((pte & mask) == (xtlb->e_tlb.tlb_pte & mask)); 788 xtlb->e_refcnt++; 789 return 0; 790 } 791 792 const int slot = e500_alloc_tlb1_entry(); 793 if (slot < 0) 794 return ENOMEM; 795 796 xtlb = &tlb1->tlb1_entries[slot]; 797 xtlb->e_tlb.tlb_va = va; 798 xtlb->e_tlb.tlb_size = len; 799 xtlb->e_tlb.tlb_pte = pte; 800 xtlb->e_tlb.tlb_asid = KERNEL_PID; 801 802 xtlb->e_hwtlb = tlb_to_hwtlb(xtlb->e_tlb); 803 xtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTIN(slot, MAS0_ESEL); 804 hwtlb_write(xtlb->e_hwtlb, true); 805 806 #if defined(MULTIPROCESSOR) 807 cpu_send_ipi(IPI_DST_NOTME, IPI_TLB1SYNC); 808 #endif 809 810 return 0; 811 } 812 813 static int 814 e500_tlb_iorelease(vaddr_t va) 815 { 816 u_int slot; 817 struct e500_xtlb * const xtlb = e500_tlb_lookup_xtlb(va, &slot); 818 819 if (xtlb == NULL) 820 return ENOENT; 821 822 if (xtlb->e_refcnt) 823 return EBUSY; 824 825 e500_free_tlb1_entry(xtlb, slot, true); 826 827 #if defined(MULTIPROCESSOR) 828 cpu_send_ipi(IPI_DST_NOTME, IPI_TLB1SYNC); 829 #endif 830 831 return 0; 832 } 833 834 static u_int 835 e500_tlbmemmap(paddr_t memstart, psize_t memsize, struct e500_tlb1 *tlb1) 836 { 837 u_int slotmask = 0; 838 u_int slots = 0, nextslot = 0; 839 KASSERT(tlb1->tlb1_numfree > 1); 840 KASSERT(((memstart + memsize - 1) & -memsize) == memstart); 841 for (paddr_t lastaddr = memstart; 0 < memsize; ) { 842 u_int cnt = __builtin_clz(memsize); 843 psize_t size = min(1UL << (31 - (cnt | 1)), tlb1->tlb1_maxsize); 844 slots += memsize / size; 845 if (slots > 4) 846 panic("%s: %d: can't map memory (%#lx) into TLB1: %s", 847 __func__, __LINE__, memsize, "too fragmented"); 848 if (slots > tlb1->tlb1_numfree - 1) 849 panic("%s: %d: can't map memory (%#lx) into TLB1: %s", 850 __func__, __LINE__, memsize, 851 "insufficent TLB entries"); 852 for (; nextslot < slots; nextslot++) { 853 const u_int freeslot = e500_alloc_tlb1_entry(); 854 struct e500_xtlb * const xtlb = 855 &tlb1->tlb1_entries[freeslot]; 856 xtlb->e_tlb.tlb_asid = KERNEL_PID; 857 xtlb->e_tlb.tlb_size = size; 858 xtlb->e_tlb.tlb_va = lastaddr; 859 xtlb->e_tlb.tlb_pte = lastaddr 860 | PTE_M | PTE_xX | PTE_xW | PTE_xR; 861 lastaddr += size; 862 memsize -= size; 863 slotmask |= 1 << (31 - freeslot); /* clz friendly */ 864 } 865 } 866 867 #if defined(MULTIPROCESSOR) 868 cpu_send_ipi(IPI_DST_NOTME, IPI_TLB1SYNC); 869 #endif 870 871 return nextslot; 872 } 873 874 static const struct tlb_md_ops e500_tlb_ops = { 875 .md_tlb_get_asid = e500_tlb_get_asid, 876 .md_tlb_set_asid = e500_tlb_set_asid, 877 .md_tlb_invalidate_all = e500_tlb_invalidate_all, 878 .md_tlb_invalidate_globals = e500_tlb_invalidate_globals, 879 .md_tlb_invalidate_asids = e500_tlb_invalidate_asids, 880 .md_tlb_invalidate_addr = e500_tlb_invalidate_addr, 881 .md_tlb_update_addr = e500_tlb_update_addr, 882 .md_tlb_record_asids = e500_tlb_record_asids, 883 .md_tlb_write_entry = e500_tlb_write_entry, 884 .md_tlb_read_entry = e500_tlb_read_entry, 885 .md_tlb_dump = e500_tlb_dump, 886 .md_tlb_walk = e500_tlb_walk, 887 }; 888 889 static const struct tlb_md_io_ops e500_tlb_io_ops = { 890 .md_tlb_mapiodev = e500_tlb_mapiodev, 891 .md_tlb_unmapiodev = e500_tlb_unmapiodev, 892 .md_tlb_ioreserve = e500_tlb_ioreserve, 893 .md_tlb_iorelease = e500_tlb_iorelease, 894 }; 895 896 void 897 e500_tlb_init(vaddr_t endkernel, psize_t memsize) 898 { 899 struct e500_tlb1 * const tlb1 = &e500_tlb1; 900 901 #if 0 902 register_t mmucfg = mfspr(SPR_MMUCFG); 903 register_t mas4 = mfspr(SPR_MAS4); 904 #endif 905 906 const uint32_t tlb1cfg = mftlb1cfg(); 907 tlb1->tlb1_numentries = TLBCFG_NENTRY(tlb1cfg); 908 KASSERT(tlb1->tlb1_numentries <= __arraycount(tlb1->tlb1_entries)); 909 /* 910 * Limit maxsize to 1G since 4G isn't really useful to us. 911 */ 912 tlb1->tlb1_minsize = 1024 << (2 * TLBCFG_MINSIZE(tlb1cfg)); 913 tlb1->tlb1_maxsize = 1024 << (2 * min(10, TLBCFG_MAXSIZE(tlb1cfg))); 914 915 #ifdef VERBOSE_INITPPC 916 printf(" tlb1cfg=%#x numentries=%u minsize=%#xKB maxsize=%#xKB", 917 tlb1cfg, tlb1->tlb1_numentries, tlb1->tlb1_minsize >> 10, 918 tlb1->tlb1_maxsize >> 10); 919 #endif 920 921 /* 922 * Let's see what's in TLB1 and we need to invalidate any entry that 923 * would fit within the kernel's mapped address space. 924 */ 925 psize_t memmapped = 0; 926 for (u_int i = 0; i < tlb1->tlb1_numentries; i++) { 927 struct e500_xtlb * const xtlb = &tlb1->tlb1_entries[i]; 928 929 xtlb->e_hwtlb = hwtlb_read(MAS0_TLBSEL_TLB1, i); 930 931 if ((xtlb->e_hwtlb.hwtlb_mas1 & MAS1_V) == 0) { 932 tlb1->tlb1_freelist[tlb1->tlb1_numfree++] = i; 933 #ifdef VERBOSE_INITPPC 934 printf(" TLB1[%u]=<unused>", i); 935 #endif 936 continue; 937 } 938 939 xtlb->e_tlb = hwtlb_to_tlb(xtlb->e_hwtlb); 940 #ifdef VERBOSE_INITPPC 941 printf(" TLB1[%u]=<%#lx,%#lx,%#x,%#x>", 942 i, xtlb->e_tlb.tlb_va, xtlb->e_tlb.tlb_size, 943 xtlb->e_tlb.tlb_asid, xtlb->e_tlb.tlb_pte); 944 #endif 945 if ((VM_MIN_KERNEL_ADDRESS <= xtlb->e_tlb.tlb_va 946 && xtlb->e_tlb.tlb_va < VM_MAX_KERNEL_ADDRESS) 947 || (xtlb->e_tlb.tlb_va < VM_MIN_KERNEL_ADDRESS 948 && VM_MIN_KERNEL_ADDRESS < 949 xtlb->e_tlb.tlb_va + xtlb->e_tlb.tlb_size)) { 950 #ifdef VERBOSE_INITPPC 951 printf("free"); 952 #endif 953 e500_free_tlb1_entry(xtlb, i, false); 954 #ifdef VERBOSE_INITPPC 955 printf("d"); 956 #endif 957 continue; 958 } 959 if ((xtlb->e_hwtlb.hwtlb_mas1 & MAS1_IPROT) == 0) { 960 xtlb->e_hwtlb.hwtlb_mas1 |= MAS1_IPROT; 961 hwtlb_write(xtlb->e_hwtlb, false); 962 #ifdef VERBOSE_INITPPC 963 printf("+iprot"); 964 #endif 965 } 966 if (xtlb->e_tlb.tlb_pte & PTE_I) 967 continue; 968 969 if (xtlb->e_tlb.tlb_va == 0 970 || xtlb->e_tlb.tlb_va + xtlb->e_tlb.tlb_size <= memsize) { 971 memmapped += xtlb->e_tlb.tlb_size; 972 /* 973 * Let make sure main memory is setup so it's memory 974 * coherent. For some reason u-boot doesn't set it up 975 * that way. 976 */ 977 if ((xtlb->e_hwtlb.hwtlb_mas2 & MAS2_M) == 0) { 978 xtlb->e_hwtlb.hwtlb_mas2 |= MAS2_M; 979 hwtlb_write(xtlb->e_hwtlb, true); 980 } 981 } 982 } 983 984 cpu_md_ops.md_tlb_ops = &e500_tlb_ops; 985 cpu_md_ops.md_tlb_io_ops = &e500_tlb_io_ops; 986 987 if (__predict_false(memmapped < memsize)) { 988 /* 989 * Let's see how many TLB entries are needed to map memory. 990 */ 991 u_int slotmask = e500_tlbmemmap(0, memsize, tlb1); 992 993 /* 994 * To map main memory into the TLB, we need to flush any 995 * existing entries from the TLB that overlap the virtual 996 * address space needed to map physical memory. That may 997 * include the entries for the pages currently used by the 998 * stack or that we are executing. So to avoid problems, we 999 * are going to temporarily map the kernel and stack into AS 1, 1000 * switch to it, and clear out the TLB entries from AS 0, 1001 * install the new TLB entries to map memory, and then switch 1002 * back to AS 0 and free the temp entry used for AS1. 1003 */ 1004 u_int b = __builtin_clz(endkernel); 1005 1006 /* 1007 * If the kernel doesn't end on a clean power of 2, we need 1008 * to round the size up (by decrementing the number of leading 1009 * zero bits). If the size isn't a power of 4KB, decrement 1010 * again to make it one. 1011 */ 1012 if (endkernel & (endkernel - 1)) 1013 b--; 1014 if ((b & 1) == 0) 1015 b--; 1016 1017 /* 1018 * Create a TLB1 mapping for the kernel in AS1. 1019 */ 1020 const u_int kslot = e500_alloc_tlb1_entry(); 1021 struct e500_xtlb * const kxtlb = &tlb1->tlb1_entries[kslot]; 1022 kxtlb->e_tlb.tlb_va = 0; 1023 kxtlb->e_tlb.tlb_size = 1UL << (31 - b); 1024 kxtlb->e_tlb.tlb_pte = PTE_M|PTE_xR|PTE_xW|PTE_xX; 1025 kxtlb->e_tlb.tlb_asid = KERNEL_PID; 1026 1027 kxtlb->e_hwtlb = tlb_to_hwtlb(kxtlb->e_tlb); 1028 kxtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTIN(kslot, MAS0_ESEL); 1029 kxtlb->e_hwtlb.hwtlb_mas1 |= MAS1_TS; 1030 hwtlb_write(kxtlb->e_hwtlb, true); 1031 1032 /* 1033 * Now that we have a TLB mapping in AS1 for the kernel and its 1034 * stack, we switch to AS1 to cleanup the TLB mappings for TLB0. 1035 */ 1036 const register_t saved_msr = mfmsr(); 1037 mtmsr(saved_msr | PSL_DS | PSL_IS); 1038 __asm volatile("isync"); 1039 1040 /* 1041 *** Invalidate all the TLB0 entries. 1042 */ 1043 e500_tlb_invalidate_all(); 1044 1045 /* 1046 *** Now let's see if we have any entries in TLB1 that would 1047 *** overlap the ones we are about to install. If so, nuke 'em. 1048 */ 1049 for (u_int i = 0; i < tlb1->tlb1_numentries; i++) { 1050 struct e500_xtlb * const xtlb = &tlb1->tlb1_entries[i]; 1051 struct e500_hwtlb * const hwtlb = &xtlb->e_hwtlb; 1052 if ((hwtlb->hwtlb_mas1 & (MAS1_V|MAS1_TS)) == MAS1_V 1053 && (hwtlb->hwtlb_mas2 & MAS2_EPN) < memsize) { 1054 e500_free_tlb1_entry(xtlb, i, false); 1055 } 1056 } 1057 1058 /* 1059 *** Now we can add the TLB entries that will map physical 1060 *** memory. If bit 0 [MSB] in slotmask is set, then tlb 1061 *** entry 0 contains a mapping for physical memory... 1062 */ 1063 struct e500_xtlb *entries = tlb1->tlb1_entries; 1064 while (slotmask != 0) { 1065 const u_int slot = __builtin_clz(slotmask); 1066 hwtlb_write(entries[slot].e_hwtlb, false); 1067 entries += slot + 1; 1068 slotmask <<= slot + 1; 1069 } 1070 1071 /* 1072 *** Synchronize the TLB and the instruction stream. 1073 */ 1074 __asm volatile("tlbsync"); 1075 __asm volatile("isync"); 1076 1077 /* 1078 *** Switch back to AS 0. 1079 */ 1080 mtmsr(saved_msr); 1081 __asm volatile("isync"); 1082 1083 /* 1084 * Free the temporary TLB1 entry. 1085 */ 1086 e500_free_tlb1_entry(kxtlb, kslot, true); 1087 } 1088 1089 /* 1090 * Finally set the MAS4 defaults. 1091 */ 1092 mtspr(SPR_MAS4, MAS4_TSIZED_4KB | MAS4_MD); 1093 1094 /* 1095 * Invalidate all the TLB0 entries. 1096 */ 1097 e500_tlb_invalidate_all(); 1098 } 1099 1100 void 1101 e500_tlb_minimize(vaddr_t endkernel) 1102 { 1103 #ifdef PMAP_MINIMALTLB 1104 struct e500_tlb1 * const tlb1 = &e500_tlb1; 1105 extern uint32_t _fdata[]; 1106 1107 u_int slot; 1108 1109 paddr_t boot_page = cpu_read_4(GUR_BPTR); 1110 if (boot_page & BPTR_EN) { 1111 /* 1112 * shift it to an address 1113 */ 1114 boot_page = (boot_page & BPTR_BOOT_PAGE) << PAGE_SHIFT; 1115 pmap_kvptefill(boot_page, boot_page + NBPG, 1116 PTE_M | PTE_xR | PTE_xW | PTE_xX); 1117 } 1118 1119 1120 KASSERT(endkernel - (uintptr_t)_fdata < 0x400000); 1121 KASSERT((uintptr_t)_fdata == 0x400000); 1122 1123 struct e500_xtlb *xtlb = e500_tlb_lookup_xtlb(endkernel, &slot); 1124 1125 KASSERT(xtlb == e500_tlb_lookup_xtlb2(0, endkernel)); 1126 const u_int tmp_slot = e500_alloc_tlb1_entry(); 1127 KASSERT(tmp_slot != (u_int) -1); 1128 1129 struct e500_xtlb * const tmp_xtlb = &tlb1->tlb1_entries[tmp_slot]; 1130 tmp_xtlb->e_tlb = xtlb->e_tlb; 1131 tmp_xtlb->e_hwtlb = tlb_to_hwtlb(tmp_xtlb->e_tlb); 1132 tmp_xtlb->e_hwtlb.hwtlb_mas1 |= MAS1_TS; 1133 KASSERT((tmp_xtlb->e_hwtlb.hwtlb_mas0 & MAS0_TLBSEL) == MAS0_TLBSEL_TLB1); 1134 tmp_xtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTIN(tmp_slot, MAS0_ESEL); 1135 hwtlb_write(tmp_xtlb->e_hwtlb, true); 1136 1137 const u_int text_slot = e500_alloc_tlb1_entry(); 1138 KASSERT(text_slot != (u_int)-1); 1139 struct e500_xtlb * const text_xtlb = &tlb1->tlb1_entries[text_slot]; 1140 text_xtlb->e_tlb.tlb_va = 0; 1141 text_xtlb->e_tlb.tlb_size = 0x400000; 1142 text_xtlb->e_tlb.tlb_pte = PTE_M | PTE_xR | PTE_xX | text_xtlb->e_tlb.tlb_va; 1143 text_xtlb->e_tlb.tlb_asid = 0; 1144 text_xtlb->e_hwtlb = tlb_to_hwtlb(text_xtlb->e_tlb); 1145 KASSERT((text_xtlb->e_hwtlb.hwtlb_mas0 & MAS0_TLBSEL) == MAS0_TLBSEL_TLB1); 1146 text_xtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTIN(text_slot, MAS0_ESEL); 1147 1148 const u_int data_slot = e500_alloc_tlb1_entry(); 1149 KASSERT(data_slot != (u_int)-1); 1150 struct e500_xtlb * const data_xtlb = &tlb1->tlb1_entries[data_slot]; 1151 data_xtlb->e_tlb.tlb_va = 0x400000; 1152 data_xtlb->e_tlb.tlb_size = 0x400000; 1153 data_xtlb->e_tlb.tlb_pte = PTE_M | PTE_xR | PTE_xW | data_xtlb->e_tlb.tlb_va; 1154 data_xtlb->e_tlb.tlb_asid = 0; 1155 data_xtlb->e_hwtlb = tlb_to_hwtlb(data_xtlb->e_tlb); 1156 KASSERT((data_xtlb->e_hwtlb.hwtlb_mas0 & MAS0_TLBSEL) == MAS0_TLBSEL_TLB1); 1157 data_xtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTIN(data_slot, MAS0_ESEL); 1158 1159 const register_t msr = mfmsr(); 1160 const register_t ts_msr = (msr | PSL_DS | PSL_IS) & ~PSL_EE; 1161 1162 __asm __volatile( 1163 "mtmsr %[ts_msr]" "\n\t" 1164 "sync" "\n\t" 1165 "isync" 1166 :: [ts_msr] "r" (ts_msr)); 1167 1168 #if 0 1169 hwtlb_write(text_xtlb->e_hwtlb, false); 1170 hwtlb_write(data_xtlb->e_hwtlb, false); 1171 e500_free_tlb1_entry(xtlb, slot, true); 1172 #endif 1173 1174 __asm __volatile( 1175 "mtmsr %[msr]" "\n\t" 1176 "sync" "\n\t" 1177 "isync" 1178 :: [msr] "r" (msr)); 1179 1180 e500_free_tlb1_entry(tmp_xtlb, tmp_slot, true); 1181 #endif /* PMAP_MINIMALTLB */ 1182 } 1183