1 /* $NetBSD: e500_tlb.c,v 1.13 2013/12/09 09:35:16 wiz Exp $ */ 2 /*- 3 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects 8 * Agency and which was developed by Matt Thomas of 3am Software Foundry. 9 * 10 * This material is based upon work supported by the Defense Advanced Research 11 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under 12 * Contract No. N66001-09-C-2073. 13 * Approved for Public Release, Distribution Unlimited 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 26 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34 * POSSIBILITY OF SUCH DAMAGE. 35 */ 36 37 #define __PMAP_PRIVATE 38 39 #include <sys/cdefs.h> 40 41 __KERNEL_RCSID(0, "$NetBSD: e500_tlb.c,v 1.13 2013/12/09 09:35:16 wiz Exp $"); 42 43 #include <sys/param.h> 44 45 #include <uvm/uvm_extern.h> 46 47 #include <powerpc/spr.h> 48 #include <powerpc/booke/spr.h> 49 #include <powerpc/booke/cpuvar.h> 50 #include <powerpc/booke/e500reg.h> 51 #include <powerpc/booke/e500var.h> 52 #include <powerpc/booke/pmap.h> 53 54 struct e500_tlb { 55 vaddr_t tlb_va; 56 uint32_t tlb_pte; 57 uint32_t tlb_asid; 58 vsize_t tlb_size; 59 }; 60 61 struct e500_hwtlb { 62 uint32_t hwtlb_mas0; 63 uint32_t hwtlb_mas1; 64 uint32_t hwtlb_mas2; 65 uint32_t hwtlb_mas3; 66 }; 67 68 struct e500_xtlb { 69 struct e500_tlb e_tlb; 70 struct e500_hwtlb e_hwtlb; 71 u_long e_refcnt; 72 }; 73 74 static struct e500_tlb1 { 75 uint32_t tlb1_maxsize; 76 uint32_t tlb1_minsize; 77 u_int tlb1_numentries; 78 u_int tlb1_numfree; 79 u_int tlb1_freelist[32]; 80 struct e500_xtlb tlb1_entries[32]; 81 } e500_tlb1; 82 83 static inline register_t mftlb0cfg(void) __pure; 84 static inline register_t mftlb1cfg(void) __pure; 85 86 static inline register_t 87 mftlb0cfg(void) 88 { 89 register_t tlb0cfg; 90 __asm("mfspr %0, %1" : "=r"(tlb0cfg) : "n"(SPR_TLB0CFG)); 91 return tlb0cfg; 92 } 93 94 static inline register_t 95 mftlb1cfg(void) 96 { 97 register_t tlb1cfg; 98 __asm("mfspr %0, %1" : "=r"(tlb1cfg) : "n"(SPR_TLB1CFG)); 99 return tlb1cfg; 100 } 101 102 static struct e500_tlb 103 hwtlb_to_tlb(const struct e500_hwtlb hwtlb) 104 { 105 struct e500_tlb tlb; 106 register_t prot_mask; 107 u_int prot_shift; 108 109 tlb.tlb_va = MAS2_EPN & hwtlb.hwtlb_mas2; 110 tlb.tlb_size = 1024 << (2 * MASX_TSIZE_GET(hwtlb.hwtlb_mas1)); 111 tlb.tlb_asid = MASX_TID_GET(hwtlb.hwtlb_mas1); 112 tlb.tlb_pte = (hwtlb.hwtlb_mas2 & MAS2_WIMGE) 113 | (hwtlb.hwtlb_mas3 & MAS3_RPN); 114 if (hwtlb.hwtlb_mas1 & MAS1_TS) { 115 prot_mask = MAS3_UX|MAS3_UW|MAS3_UR; 116 prot_shift = PTE_RWX_SHIFT - 1; 117 } else { 118 prot_mask = MAS3_SX|MAS3_SW|MAS3_SR; 119 prot_shift = PTE_RWX_SHIFT; 120 } 121 tlb.tlb_pte |= (prot_mask & hwtlb.hwtlb_mas3) << prot_shift; 122 return tlb; 123 } 124 125 static inline struct e500_hwtlb 126 hwtlb_read(uint32_t mas0, u_int slot) 127 { 128 struct e500_hwtlb hwtlb; 129 register_t tlbcfg; 130 131 if (__predict_true(mas0 == MAS0_TLBSEL_TLB0)) { 132 tlbcfg = mftlb0cfg(); 133 } else if (mas0 == MAS0_TLBSEL_TLB1) { 134 tlbcfg = mftlb1cfg(); 135 } else { 136 panic("%s:%d: unexpected MAS0 %#" PRIx32, 137 __func__, __LINE__, mas0); 138 } 139 140 /* 141 * ESEL is the way we want to look up. 142 * If tlbassoc is the same as tlbentries (like in TLB1) then the TLB is 143 * fully associative, the entire slot is placed into ESEL. If tlbassoc 144 * is less than the number of tlb entries, the slot is split in two 145 * fields. Since the TLB is M rows by N ways, the lowers bits are for 146 * row (MAS2[EPN]) and the upper for the way (MAS1[ESEL]). 147 */ 148 const u_int tlbassoc = TLBCFG_ASSOC(tlbcfg); 149 const u_int tlbentries = TLBCFG_NENTRY(tlbcfg); 150 const u_int esel_shift = 151 __builtin_clz(tlbassoc) - __builtin_clz(tlbentries); 152 153 /* 154 * Disable interrupts since we don't want anyone else mucking with 155 * the MMU Assist registers 156 */ 157 const register_t msr = wrtee(0); 158 const register_t saved_mas0 = mfspr(SPR_MAS0); 159 mtspr(SPR_MAS0, mas0 | MAS0_ESEL_MAKE(slot >> esel_shift)); 160 161 if (__predict_true(tlbassoc > tlbentries)) 162 mtspr(SPR_MAS2, slot << PAGE_SHIFT); 163 164 /* 165 * Now select the entry and grab its contents. 166 */ 167 __asm volatile("tlbre"); 168 169 hwtlb.hwtlb_mas0 = mfspr(SPR_MAS0); 170 hwtlb.hwtlb_mas1 = mfspr(SPR_MAS1); 171 hwtlb.hwtlb_mas2 = mfspr(SPR_MAS2); 172 hwtlb.hwtlb_mas3 = mfspr(SPR_MAS3); 173 174 mtspr(SPR_MAS0, saved_mas0); 175 wrtee(msr); /* restore interrupts */ 176 177 return hwtlb; 178 } 179 180 static inline void 181 hwtlb_write(const struct e500_hwtlb hwtlb, bool needs_sync) 182 { 183 const register_t msr = wrtee(0); 184 const uint32_t saved_mas0 = mfspr(SPR_MAS0); 185 186 /* 187 * Need to always write MAS0 and MAS1 188 */ 189 mtspr(SPR_MAS0, hwtlb.hwtlb_mas0); 190 mtspr(SPR_MAS1, hwtlb.hwtlb_mas1); 191 192 /* 193 * Only write the VPN/WIMGE if this is in TLB0 or if a valid mapping. 194 */ 195 if ((hwtlb.hwtlb_mas0 & MAS0_TLBSEL) == MAS0_TLBSEL_TLB0 196 || (hwtlb.hwtlb_mas1 & MAS1_V)) { 197 mtspr(SPR_MAS2, hwtlb.hwtlb_mas2); 198 } 199 /* 200 * Only need to write the RPN/prot if we are dealing with a valid 201 * mapping. 202 */ 203 if (hwtlb.hwtlb_mas1 & MAS1_V) { 204 mtspr(SPR_MAS3, hwtlb.hwtlb_mas3); 205 //mtspr(SPR_MAS7, 0); 206 } 207 208 #if 0 209 printf("%s->[%x,%x,%x,%x]\n", 210 __func__, 211 hwtlb.hwtlb_mas0, hwtlb.hwtlb_mas1, 212 hwtlb.hwtlb_mas2, hwtlb.hwtlb_mas3); 213 #endif 214 __asm volatile("tlbwe"); 215 if (needs_sync) { 216 __asm volatile("tlbsync\n\tisync\n\tsync"); 217 } 218 219 mtspr(SPR_MAS0, saved_mas0); 220 wrtee(msr); 221 } 222 223 static struct e500_hwtlb 224 tlb_to_hwtlb(const struct e500_tlb tlb) 225 { 226 struct e500_hwtlb hwtlb; 227 228 KASSERT(trunc_page(tlb.tlb_va) == tlb.tlb_va); 229 KASSERT(tlb.tlb_size != 0); 230 KASSERT((tlb.tlb_size & (tlb.tlb_size - 1)) == 0); 231 const uint32_t prot_mask = tlb.tlb_pte & PTE_RWX_MASK; 232 if (__predict_true(tlb.tlb_size == PAGE_SIZE)) { 233 hwtlb.hwtlb_mas0 = 0; 234 hwtlb.hwtlb_mas1 = MAS1_V | MASX_TSIZE_MAKE(1); 235 /* 236 * A non-zero ASID means this is a user page so mark it as 237 * being in the user's address space. 238 */ 239 if (tlb.tlb_asid) { 240 hwtlb.hwtlb_mas1 |= MAS1_TS 241 | MASX_TID_MAKE(tlb.tlb_asid); 242 hwtlb.hwtlb_mas3 = (prot_mask >> (PTE_RWX_SHIFT - 1)) 243 | ((prot_mask & ~PTE_xX) >> PTE_RWX_SHIFT); 244 KASSERT(prot_mask & PTE_xR); 245 KASSERT(hwtlb.hwtlb_mas3 & MAS3_UR); 246 CTASSERT(MAS3_UR == (PTE_xR >> (PTE_RWX_SHIFT - 1))); 247 CTASSERT(MAS3_SR == (PTE_xR >> PTE_RWX_SHIFT)); 248 } else { 249 hwtlb.hwtlb_mas3 = prot_mask >> PTE_RWX_SHIFT; 250 } 251 if (tlb.tlb_pte & PTE_UNMODIFIED) 252 hwtlb.hwtlb_mas3 &= ~(MAS3_UW|MAS3_SW); 253 if (tlb.tlb_pte & PTE_UNSYNCED) 254 hwtlb.hwtlb_mas3 &= ~(MAS3_UX|MAS3_SX); 255 } else { 256 KASSERT(tlb.tlb_asid == 0); 257 KASSERT((tlb.tlb_size & 0xaaaaa7ff) == 0); 258 u_int cntlz = __builtin_clz(tlb.tlb_size); 259 KASSERT(cntlz & 1); 260 KASSERT(cntlz <= 19); 261 hwtlb.hwtlb_mas0 = MAS0_TLBSEL_TLB1; 262 /* 263 * TSIZE is defined (4^TSIZE) Kbytes except a TSIZE of 0 is not 264 * allowed. So 1K would be 0x00000400 giving 21 leading zero 265 * bits. Subtracting the leading number of zero bits from 21 266 * and dividing by 2 gives us the number that the MMU wants. 267 */ 268 hwtlb.hwtlb_mas1 = MASX_TSIZE_MAKE(((31 - 10) - cntlz) / 2) 269 | MAS1_IPROT | MAS1_V; 270 hwtlb.hwtlb_mas3 = prot_mask >> PTE_RWX_SHIFT; 271 } 272 /* We are done with MAS1, on to MAS2 ... */ 273 hwtlb.hwtlb_mas2 = tlb.tlb_va | (tlb.tlb_pte & PTE_WIMGE_MASK); 274 hwtlb.hwtlb_mas3 |= tlb.tlb_pte & PTE_RPN_MASK; 275 276 return hwtlb; 277 } 278 279 void * 280 e500_tlb1_fetch(size_t slot) 281 { 282 struct e500_tlb1 * const tlb1 = &e500_tlb1; 283 284 return &tlb1->tlb1_entries[slot].e_hwtlb; 285 } 286 287 void 288 e500_tlb1_sync(void) 289 { 290 struct e500_tlb1 * const tlb1 = &e500_tlb1; 291 for (u_int slot = 1; slot < tlb1->tlb1_numentries; slot++) { 292 const struct e500_hwtlb * const new_hwtlb = 293 &tlb1->tlb1_entries[slot].e_hwtlb; 294 const struct e500_hwtlb old_hwtlb = 295 hwtlb_read(MAS0_TLBSEL_TLB1, slot); 296 #define CHANGED(n,o,f) ((n)->f != (o).f) 297 bool mas1_changed_p = CHANGED(new_hwtlb, old_hwtlb, hwtlb_mas1); 298 bool mas2_changed_p = CHANGED(new_hwtlb, old_hwtlb, hwtlb_mas2); 299 bool mas3_changed_p = CHANGED(new_hwtlb, old_hwtlb, hwtlb_mas3); 300 #undef CHANGED 301 bool new_valid_p = (new_hwtlb->hwtlb_mas1 & MAS1_V) != 0; 302 bool old_valid_p = (old_hwtlb.hwtlb_mas1 & MAS1_V) != 0; 303 if ((new_valid_p || old_valid_p) 304 && (mas1_changed_p 305 || (new_valid_p 306 && (mas2_changed_p || mas3_changed_p)))) 307 hwtlb_write(*new_hwtlb, true); 308 } 309 } 310 311 static int 312 e500_alloc_tlb1_entry(void) 313 { 314 struct e500_tlb1 * const tlb1 = &e500_tlb1; 315 316 if (tlb1->tlb1_numfree == 0) 317 return -1; 318 const u_int slot = tlb1->tlb1_freelist[--tlb1->tlb1_numfree]; 319 KASSERT((tlb1->tlb1_entries[slot].e_hwtlb.hwtlb_mas1 & MAS1_V) == 0); 320 tlb1->tlb1_entries[slot].e_hwtlb.hwtlb_mas0 = 321 MAS0_TLBSEL_TLB1 | __SHIFTIN(slot, MAS0_ESEL); 322 return (int)slot; 323 } 324 325 static void 326 e500_free_tlb1_entry(struct e500_xtlb *xtlb, u_int slot, bool needs_sync) 327 { 328 struct e500_tlb1 * const tlb1 = &e500_tlb1; 329 KASSERT(slot < tlb1->tlb1_numentries); 330 KASSERT(&tlb1->tlb1_entries[slot] == xtlb); 331 332 KASSERT(xtlb->e_hwtlb.hwtlb_mas0 == (MAS0_TLBSEL_TLB1|__SHIFTIN(slot, MAS0_ESEL))); 333 xtlb->e_hwtlb.hwtlb_mas1 &= ~(MAS1_V|MAS1_IPROT); 334 hwtlb_write(xtlb->e_hwtlb, needs_sync); 335 336 const register_t msr = wrtee(0); 337 tlb1->tlb1_freelist[tlb1->tlb1_numfree++] = slot; 338 wrtee(msr); 339 } 340 341 static tlb_asid_t 342 e500_tlb_get_asid(void) 343 { 344 return mfspr(SPR_PID0); 345 } 346 347 static void 348 e500_tlb_set_asid(tlb_asid_t asid) 349 { 350 mtspr(SPR_PID0, asid); 351 } 352 353 static void 354 e500_tlb_invalidate_all(void) 355 { 356 /* 357 * This does a flash invalidate of all entries in TLB0. 358 * We don't touch TLB1 since we don't expect those to be volatile. 359 */ 360 #if 1 361 __asm volatile("tlbivax\t0, %0" :: "b"(4)); /* INV_ALL */ 362 __asm volatile("tlbsync\n\tisync\n\tsync"); 363 #else 364 mtspr(SPR_MMUCSR0, MMUCSR0_TLB0_FL); 365 while (mfspr(SPR_MMUCSR0) != 0) 366 ; 367 #endif 368 } 369 370 static void 371 e500_tlb_invalidate_globals(void) 372 { 373 const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg()); 374 const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg()); 375 const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT; 376 const vaddr_t kstack_lo = (uintptr_t)curlwp->l_addr; 377 const vaddr_t kstack_hi = kstack_lo + USPACE - 1; 378 const vaddr_t epn_kstack_lo = kstack_lo & (max_epn - 1); 379 const vaddr_t epn_kstack_hi = kstack_hi & (max_epn - 1); 380 381 const register_t msr = wrtee(0); 382 for (size_t assoc = 0; assoc < tlbassoc; assoc++) { 383 mtspr(SPR_MAS0, MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0); 384 for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) { 385 mtspr(SPR_MAS2, epn); 386 __asm volatile("tlbre"); 387 uint32_t mas1 = mfspr(SPR_MAS1); 388 389 /* 390 * Make sure this is a valid kernel entry first. 391 */ 392 if ((mas1 & (MAS1_V|MAS1_TID|MAS1_TS)) != MAS1_V) 393 continue; 394 395 /* 396 * We have a valid kernel TLB entry. But if it matches 397 * the stack we are currently running on, it would 398 * unwise to invalidate it. First see if the epn 399 * overlaps the stack. If it does then get the 400 * VA and see if it really is part of the stack. 401 */ 402 if (epn_kstack_lo < epn_kstack_hi 403 ? (epn_kstack_lo <= epn && epn <= epn_kstack_hi) 404 : (epn <= epn_kstack_hi || epn_kstack_lo <= epn)) { 405 const uint32_t mas2_epn = 406 mfspr(SPR_MAS2) & MAS2_EPN; 407 if (kstack_lo <= mas2_epn 408 && mas2_epn <= kstack_hi) 409 continue; 410 } 411 mtspr(SPR_MAS1, mas1 ^ MAS1_V); 412 __asm volatile("tlbwe"); 413 } 414 } 415 __asm volatile("isync\n\tsync"); 416 wrtee(msr); 417 } 418 419 static void 420 e500_tlb_invalidate_asids(tlb_asid_t asid_lo, tlb_asid_t asid_hi) 421 { 422 const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg()); 423 const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg()); 424 const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT; 425 426 asid_lo = __SHIFTIN(asid_lo, MAS1_TID); 427 asid_hi = __SHIFTIN(asid_hi, MAS1_TID); 428 429 const register_t msr = wrtee(0); 430 for (size_t assoc = 0; assoc < tlbassoc; assoc++) { 431 mtspr(SPR_MAS0, MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0); 432 for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) { 433 mtspr(SPR_MAS2, epn); 434 __asm volatile("tlbre"); 435 const uint32_t mas1 = mfspr(SPR_MAS1); 436 /* 437 * If this is a valid entry for AS space 1 and 438 * its asid matches the constraints of the caller, 439 * clear its valid bit. 440 */ 441 if ((mas1 & (MAS1_V|MAS1_TS)) == (MAS1_V|MAS1_TS) 442 && asid_lo <= (mas1 & MAS1_TID) 443 && (mas1 & MAS1_TID) <= asid_hi) { 444 mtspr(SPR_MAS1, mas1 ^ MAS1_V); 445 #if 0 446 printf("%s[%zu,%zu]->[%x]\n", 447 __func__, assoc, epn, mas1); 448 #endif 449 __asm volatile("tlbwe"); 450 } 451 } 452 } 453 __asm volatile("isync\n\tsync"); 454 wrtee(msr); 455 } 456 457 static u_int 458 e500_tlb_record_asids(u_long *bitmap) 459 { 460 const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg()); 461 const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg()); 462 const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT; 463 const size_t nbits = 8 * sizeof(bitmap[0]); 464 u_int found = 0; 465 466 const register_t msr = wrtee(0); 467 for (size_t assoc = 0; assoc < tlbassoc; assoc++) { 468 mtspr(SPR_MAS0, MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0); 469 for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) { 470 mtspr(SPR_MAS2, epn); 471 __asm volatile("tlbre"); 472 const uint32_t mas1 = mfspr(SPR_MAS1); 473 /* 474 * If this is a valid entry for AS space 1 and 475 * its asid matches the constraints of the caller, 476 * clear its valid bit. 477 */ 478 if ((mas1 & (MAS1_V|MAS1_TS)) == (MAS1_V|MAS1_TS)) { 479 const uint32_t asid = MASX_TID_GET(mas1); 480 const u_int i = asid / nbits; 481 const u_long mask = 1UL << (asid & (nbits - 1)); 482 if ((bitmap[i] & mask) == 0) { 483 bitmap[i] |= mask; 484 found++; 485 } 486 } 487 } 488 } 489 wrtee(msr); 490 491 return found; 492 } 493 494 static void 495 e500_tlb_invalidate_addr(vaddr_t va, tlb_asid_t asid) 496 { 497 KASSERT((va & PAGE_MASK) == 0); 498 /* 499 * Bits 60 & 61 have meaning 500 */ 501 if (asid == KERNEL_PID) { 502 /* 503 * For data accesses, the context-synchronizing instruction 504 * before tlbwe or tlbivax ensures that all memory accesses 505 * due to preceding instructions have completed to a point 506 * at which they have reported all exceptions they will cause. 507 */ 508 __asm volatile("isync"); 509 } 510 __asm volatile("tlbivax\t0, %0" :: "b"(va)); 511 __asm volatile("tlbsync"); 512 __asm volatile("tlbsync"); /* Why? */ 513 if (asid == KERNEL_PID) { 514 /* 515 * The context-synchronizing instruction after tlbwe or tlbivax 516 * ensures that subsequent accesses (data and instruction) use 517 * the updated value in any TLB entries affected. 518 */ 519 __asm volatile("isync\n\tsync"); 520 } 521 } 522 523 static bool 524 e500_tlb_update_addr(vaddr_t va, tlb_asid_t asid, pt_entry_t pte, bool insert) 525 { 526 struct e500_hwtlb hwtlb = tlb_to_hwtlb( 527 (struct e500_tlb){ .tlb_va = va, .tlb_asid = asid, 528 .tlb_size = PAGE_SIZE, .tlb_pte = pte,}); 529 530 register_t msr = wrtee(0); 531 mtspr(SPR_MAS6, asid ? __SHIFTIN(asid, MAS6_SPID0) | MAS6_SAS : 0); 532 __asm volatile("tlbsx 0, %0" :: "b"(va)); 533 register_t mas1 = mfspr(SPR_MAS1); 534 if ((mas1 & MAS1_V) == 0) { 535 if (!insert) { 536 wrtee(msr); 537 #if 0 538 printf("%s(%#lx,%#x,%#x,%x)<no update>\n", 539 __func__, va, asid, pte, insert); 540 #endif 541 return false; 542 } 543 mtspr(SPR_MAS1, hwtlb.hwtlb_mas1); 544 } 545 mtspr(SPR_MAS2, hwtlb.hwtlb_mas2); 546 mtspr(SPR_MAS3, hwtlb.hwtlb_mas3); 547 //mtspr(SPR_MAS7, 0); 548 __asm volatile("tlbwe"); 549 if (asid == KERNEL_PID) 550 __asm volatile("isync\n\tsync"); 551 wrtee(msr); 552 #if 0 553 if (asid) 554 printf("%s(%#lx,%#x,%#x,%x)->[%x,%x,%x]\n", 555 __func__, va, asid, pte, insert, 556 hwtlb.hwtlb_mas1, hwtlb.hwtlb_mas2, hwtlb.hwtlb_mas3); 557 #endif 558 return (mas1 & MAS1_V) != 0; 559 } 560 561 static void 562 e500_tlb_write_entry(size_t index, const struct tlbmask *tlb) 563 { 564 } 565 566 static void 567 e500_tlb_read_entry(size_t index, struct tlbmask *tlb) 568 { 569 } 570 571 static void 572 e500_tlb_dump(void (*pr)(const char *, ...)) 573 { 574 const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg()); 575 const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg()); 576 const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT; 577 const uint32_t saved_mas0 = mfspr(SPR_MAS0); 578 size_t valid = 0; 579 580 if (pr == NULL) 581 pr = printf; 582 583 const register_t msr = wrtee(0); 584 for (size_t assoc = 0; assoc < tlbassoc; assoc++) { 585 struct e500_hwtlb hwtlb; 586 hwtlb.hwtlb_mas0 = MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0; 587 mtspr(SPR_MAS0, hwtlb.hwtlb_mas0); 588 for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) { 589 mtspr(SPR_MAS2, epn); 590 __asm volatile("tlbre"); 591 hwtlb.hwtlb_mas1 = mfspr(SPR_MAS1); 592 /* 593 * If this is a valid entry for AS space 1 and 594 * its asid matches the constraints of the caller, 595 * clear its valid bit. 596 */ 597 if (hwtlb.hwtlb_mas1 & MAS1_V) { 598 hwtlb.hwtlb_mas2 = mfspr(SPR_MAS2); 599 hwtlb.hwtlb_mas3 = mfspr(SPR_MAS3); 600 struct e500_tlb tlb = hwtlb_to_tlb(hwtlb); 601 (*pr)("[%zu,%zu]->[%x,%x,%x]", 602 assoc, atop(epn), 603 hwtlb.hwtlb_mas1, 604 hwtlb.hwtlb_mas2, 605 hwtlb.hwtlb_mas3); 606 (*pr)(": VA=%#lx size=4KB asid=%u pte=%x", 607 tlb.tlb_va, tlb.tlb_asid, tlb.tlb_pte); 608 (*pr)(" (RPN=%#x,%s%s%s%s%s,%s%s%s%s%s)\n", 609 tlb.tlb_pte & PTE_RPN_MASK, 610 tlb.tlb_pte & PTE_xR ? "R" : "", 611 tlb.tlb_pte & PTE_xW ? "W" : "", 612 tlb.tlb_pte & PTE_UNMODIFIED ? "*" : "", 613 tlb.tlb_pte & PTE_xX ? "X" : "", 614 tlb.tlb_pte & PTE_UNSYNCED ? "*" : "", 615 tlb.tlb_pte & PTE_W ? "W" : "", 616 tlb.tlb_pte & PTE_I ? "I" : "", 617 tlb.tlb_pte & PTE_M ? "M" : "", 618 tlb.tlb_pte & PTE_G ? "G" : "", 619 tlb.tlb_pte & PTE_E ? "E" : ""); 620 valid++; 621 } 622 } 623 } 624 mtspr(SPR_MAS0, saved_mas0); 625 wrtee(msr); 626 (*pr)("%s: %zu valid entries\n", __func__, valid); 627 } 628 629 static void 630 e500_tlb_walk(void *ctx, bool (*func)(void *, vaddr_t, uint32_t, uint32_t)) 631 { 632 const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg()); 633 const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg()); 634 const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT; 635 const uint32_t saved_mas0 = mfspr(SPR_MAS0); 636 637 const register_t msr = wrtee(0); 638 for (size_t assoc = 0; assoc < tlbassoc; assoc++) { 639 struct e500_hwtlb hwtlb; 640 hwtlb.hwtlb_mas0 = MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0; 641 mtspr(SPR_MAS0, hwtlb.hwtlb_mas0); 642 for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) { 643 mtspr(SPR_MAS2, epn); 644 __asm volatile("tlbre"); 645 hwtlb.hwtlb_mas1 = mfspr(SPR_MAS1); 646 if (hwtlb.hwtlb_mas1 & MAS1_V) { 647 hwtlb.hwtlb_mas2 = mfspr(SPR_MAS2); 648 hwtlb.hwtlb_mas3 = mfspr(SPR_MAS3); 649 struct e500_tlb tlb = hwtlb_to_tlb(hwtlb); 650 if (!(*func)(ctx, tlb.tlb_va, tlb.tlb_asid, 651 tlb.tlb_pte)) 652 break; 653 } 654 } 655 } 656 mtspr(SPR_MAS0, saved_mas0); 657 wrtee(msr); 658 } 659 660 static struct e500_xtlb * 661 e500_tlb_lookup_xtlb_pa(vaddr_t pa, u_int *slotp) 662 { 663 struct e500_tlb1 * const tlb1 = &e500_tlb1; 664 struct e500_xtlb *xtlb = tlb1->tlb1_entries; 665 666 /* 667 * See if we have a TLB entry for the pa. 668 */ 669 for (u_int i = 0; i < tlb1->tlb1_numentries; i++, xtlb++) { 670 psize_t mask = ~(xtlb->e_tlb.tlb_size - 1); 671 if ((xtlb->e_hwtlb.hwtlb_mas1 & MAS1_V) 672 && ((pa ^ xtlb->e_tlb.tlb_pte) & mask) == 0) { 673 if (slotp != NULL) 674 *slotp = i; 675 return xtlb; 676 } 677 } 678 679 return NULL; 680 } 681 682 struct e500_xtlb * 683 e500_tlb_lookup_xtlb(vaddr_t va, u_int *slotp) 684 { 685 struct e500_tlb1 * const tlb1 = &e500_tlb1; 686 struct e500_xtlb *xtlb = tlb1->tlb1_entries; 687 688 /* 689 * See if we have a TLB entry for the va. 690 */ 691 for (u_int i = 0; i < tlb1->tlb1_numentries; i++, xtlb++) { 692 vsize_t mask = ~(xtlb->e_tlb.tlb_size - 1); 693 if ((xtlb->e_hwtlb.hwtlb_mas1 & MAS1_V) 694 && ((va ^ xtlb->e_tlb.tlb_va) & mask) == 0) { 695 if (slotp != NULL) 696 *slotp = i; 697 return xtlb; 698 } 699 } 700 701 return NULL; 702 } 703 704 static struct e500_xtlb * 705 e500_tlb_lookup_xtlb2(vaddr_t va, vsize_t len) 706 { 707 struct e500_tlb1 * const tlb1 = &e500_tlb1; 708 struct e500_xtlb *xtlb = tlb1->tlb1_entries; 709 710 /* 711 * See if we have a TLB entry for the pa. 712 */ 713 for (u_int i = 0; i < tlb1->tlb1_numentries; i++, xtlb++) { 714 vsize_t mask = ~(xtlb->e_tlb.tlb_size - 1); 715 if ((xtlb->e_hwtlb.hwtlb_mas1 & MAS1_V) 716 && ((va ^ xtlb->e_tlb.tlb_va) & mask) == 0 717 && (((va + len - 1) ^ va) & mask) == 0) { 718 return xtlb; 719 } 720 } 721 722 return NULL; 723 } 724 725 static void * 726 e500_tlb_mapiodev(paddr_t pa, psize_t len, bool prefetchable) 727 { 728 struct e500_xtlb * const xtlb = e500_tlb_lookup_xtlb_pa(pa, NULL); 729 730 /* 731 * See if we have a TLB entry for the pa. If completely falls within 732 * mark the reference and return the pa. But only if the tlb entry 733 * is not cacheable. 734 */ 735 if (xtlb 736 && (prefetchable 737 || (xtlb->e_tlb.tlb_pte & PTE_WIG) == (PTE_I|PTE_G))) { 738 xtlb->e_refcnt++; 739 return (void *) (xtlb->e_tlb.tlb_va 740 + pa - (xtlb->e_tlb.tlb_pte & PTE_RPN_MASK)); 741 } 742 return NULL; 743 } 744 745 static void 746 e500_tlb_unmapiodev(vaddr_t va, vsize_t len) 747 { 748 if (va < VM_MIN_KERNEL_ADDRESS || VM_MAX_KERNEL_ADDRESS <= va) { 749 struct e500_xtlb * const xtlb = e500_tlb_lookup_xtlb(va, NULL); 750 if (xtlb) 751 xtlb->e_refcnt--; 752 } 753 } 754 755 static int 756 e500_tlb_ioreserve(vaddr_t va, vsize_t len, pt_entry_t pte) 757 { 758 struct e500_tlb1 * const tlb1 = &e500_tlb1; 759 struct e500_xtlb *xtlb; 760 761 KASSERT(len & 0x55555000); 762 KASSERT((len & ~0x55555000) == 0); 763 KASSERT(len >= PAGE_SIZE); 764 KASSERT((len & (len - 1)) == 0); 765 KASSERT((va & (len - 1)) == 0); 766 KASSERT(((pte & PTE_RPN_MASK) & (len - 1)) == 0); 767 768 if ((xtlb = e500_tlb_lookup_xtlb2(va, len)) != NULL) { 769 psize_t mask = ~(xtlb->e_tlb.tlb_size - 1); 770 KASSERT(len <= xtlb->e_tlb.tlb_size); 771 KASSERT((pte & mask) == (xtlb->e_tlb.tlb_pte & mask)); 772 xtlb->e_refcnt++; 773 return 0; 774 } 775 776 const int slot = e500_alloc_tlb1_entry(); 777 if (slot < 0) 778 return ENOMEM; 779 780 xtlb = &tlb1->tlb1_entries[slot]; 781 xtlb->e_tlb.tlb_va = va; 782 xtlb->e_tlb.tlb_size = len; 783 xtlb->e_tlb.tlb_pte = pte; 784 xtlb->e_tlb.tlb_asid = KERNEL_PID; 785 786 xtlb->e_hwtlb = tlb_to_hwtlb(xtlb->e_tlb); 787 xtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTIN(slot, MAS0_ESEL); 788 hwtlb_write(xtlb->e_hwtlb, true); 789 return 0; 790 } 791 792 static int 793 e500_tlb_iorelease(vaddr_t va) 794 { 795 u_int slot; 796 struct e500_xtlb * const xtlb = e500_tlb_lookup_xtlb(va, &slot); 797 798 if (xtlb == NULL) 799 return ENOENT; 800 801 if (xtlb->e_refcnt) 802 return EBUSY; 803 804 e500_free_tlb1_entry(xtlb, slot, true); 805 806 return 0; 807 } 808 809 static u_int 810 e500_tlbmemmap(paddr_t memstart, psize_t memsize, struct e500_tlb1 *tlb1) 811 { 812 u_int slotmask = 0; 813 u_int slots = 0, nextslot = 0; 814 KASSERT(tlb1->tlb1_numfree > 1); 815 KASSERT(((memstart + memsize - 1) & -memsize) == memstart); 816 for (paddr_t lastaddr = memstart; 0 < memsize; ) { 817 u_int cnt = __builtin_clz(memsize); 818 psize_t size = min(1UL << (31 - (cnt | 1)), tlb1->tlb1_maxsize); 819 slots += memsize / size; 820 if (slots > 4) 821 panic("%s: %d: can't map memory (%#lx) into TLB1: %s", 822 __func__, __LINE__, memsize, "too fragmented"); 823 if (slots > tlb1->tlb1_numfree - 1) 824 panic("%s: %d: can't map memory (%#lx) into TLB1: %s", 825 __func__, __LINE__, memsize, 826 "insufficent TLB entries"); 827 for (; nextslot < slots; nextslot++) { 828 const u_int freeslot = e500_alloc_tlb1_entry(); 829 struct e500_xtlb * const xtlb = 830 &tlb1->tlb1_entries[freeslot]; 831 xtlb->e_tlb.tlb_asid = KERNEL_PID; 832 xtlb->e_tlb.tlb_size = size; 833 xtlb->e_tlb.tlb_va = lastaddr; 834 xtlb->e_tlb.tlb_pte = lastaddr 835 | PTE_M | PTE_xX | PTE_xW | PTE_xR; 836 lastaddr += size; 837 memsize -= size; 838 slotmask |= 1 << (31 - freeslot); /* clz friendly */ 839 } 840 } 841 842 return nextslot; 843 } 844 static const struct tlb_md_ops e500_tlb_ops = { 845 .md_tlb_get_asid = e500_tlb_get_asid, 846 .md_tlb_set_asid = e500_tlb_set_asid, 847 .md_tlb_invalidate_all = e500_tlb_invalidate_all, 848 .md_tlb_invalidate_globals = e500_tlb_invalidate_globals, 849 .md_tlb_invalidate_asids = e500_tlb_invalidate_asids, 850 .md_tlb_invalidate_addr = e500_tlb_invalidate_addr, 851 .md_tlb_update_addr = e500_tlb_update_addr, 852 .md_tlb_record_asids = e500_tlb_record_asids, 853 .md_tlb_write_entry = e500_tlb_write_entry, 854 .md_tlb_read_entry = e500_tlb_read_entry, 855 .md_tlb_dump = e500_tlb_dump, 856 .md_tlb_walk = e500_tlb_walk, 857 }; 858 859 static const struct tlb_md_io_ops e500_tlb_io_ops = { 860 .md_tlb_mapiodev = e500_tlb_mapiodev, 861 .md_tlb_unmapiodev = e500_tlb_unmapiodev, 862 .md_tlb_ioreserve = e500_tlb_ioreserve, 863 .md_tlb_iorelease = e500_tlb_iorelease, 864 }; 865 866 void 867 e500_tlb_init(vaddr_t endkernel, psize_t memsize) 868 { 869 struct e500_tlb1 * const tlb1 = &e500_tlb1; 870 871 #if 0 872 register_t mmucfg = mfspr(SPR_MMUCFG); 873 register_t mas4 = mfspr(SPR_MAS4); 874 #endif 875 876 const uint32_t tlb1cfg = mftlb1cfg(); 877 tlb1->tlb1_numentries = TLBCFG_NENTRY(tlb1cfg); 878 KASSERT(tlb1->tlb1_numentries <= __arraycount(tlb1->tlb1_entries)); 879 /* 880 * Limit maxsize to 1G since 4G isn't really useful to us. 881 */ 882 tlb1->tlb1_minsize = 1024 << (2 * TLBCFG_MINSIZE(tlb1cfg)); 883 tlb1->tlb1_maxsize = 1024 << (2 * min(10, TLBCFG_MAXSIZE(tlb1cfg))); 884 885 #ifdef VERBOSE_INITPPC 886 printf(" tlb1cfg=%#x numentries=%u minsize=%#xKB maxsize=%#xKB", 887 tlb1cfg, tlb1->tlb1_numentries, tlb1->tlb1_minsize >> 10, 888 tlb1->tlb1_maxsize >> 10); 889 #endif 890 891 /* 892 * Let's see what's in TLB1 and we need to invalidate any entry that 893 * would fit within the kernel's mapped address space. 894 */ 895 psize_t memmapped = 0; 896 for (u_int i = 0; i < tlb1->tlb1_numentries; i++) { 897 struct e500_xtlb * const xtlb = &tlb1->tlb1_entries[i]; 898 899 xtlb->e_hwtlb = hwtlb_read(MAS0_TLBSEL_TLB1, i); 900 901 if ((xtlb->e_hwtlb.hwtlb_mas1 & MAS1_V) == 0) { 902 tlb1->tlb1_freelist[tlb1->tlb1_numfree++] = i; 903 #ifdef VERBOSE_INITPPC 904 printf(" TLB1[%u]=<unused>", i); 905 #endif 906 continue; 907 } 908 909 xtlb->e_tlb = hwtlb_to_tlb(xtlb->e_hwtlb); 910 #ifdef VERBOSE_INITPPC 911 printf(" TLB1[%u]=<%#lx,%#lx,%#x,%#x>", 912 i, xtlb->e_tlb.tlb_va, xtlb->e_tlb.tlb_size, 913 xtlb->e_tlb.tlb_asid, xtlb->e_tlb.tlb_pte); 914 #endif 915 if ((VM_MIN_KERNEL_ADDRESS <= xtlb->e_tlb.tlb_va 916 && xtlb->e_tlb.tlb_va < VM_MAX_KERNEL_ADDRESS) 917 || (xtlb->e_tlb.tlb_va < VM_MIN_KERNEL_ADDRESS 918 && VM_MIN_KERNEL_ADDRESS < 919 xtlb->e_tlb.tlb_va + xtlb->e_tlb.tlb_size)) { 920 #ifdef VERBOSE_INITPPC 921 printf("free"); 922 #endif 923 e500_free_tlb1_entry(xtlb, i, false); 924 #ifdef VERBOSE_INITPPC 925 printf("d"); 926 #endif 927 continue; 928 } 929 if ((xtlb->e_hwtlb.hwtlb_mas1 & MAS1_IPROT) == 0) { 930 xtlb->e_hwtlb.hwtlb_mas1 |= MAS1_IPROT; 931 hwtlb_write(xtlb->e_hwtlb, false); 932 #ifdef VERBOSE_INITPPC 933 printf("+iprot"); 934 #endif 935 } 936 if (xtlb->e_tlb.tlb_pte & PTE_I) 937 continue; 938 939 if (xtlb->e_tlb.tlb_va == 0 940 || xtlb->e_tlb.tlb_va + xtlb->e_tlb.tlb_size <= memsize) { 941 memmapped += xtlb->e_tlb.tlb_size; 942 /* 943 * Let make sure main memory is setup so it's memory 944 * coherent. For some reason u-boot doesn't set it up 945 * that way. 946 */ 947 if ((xtlb->e_hwtlb.hwtlb_mas2 & MAS2_M) == 0) { 948 xtlb->e_hwtlb.hwtlb_mas2 |= MAS2_M; 949 hwtlb_write(xtlb->e_hwtlb, true); 950 } 951 } 952 } 953 954 cpu_md_ops.md_tlb_ops = &e500_tlb_ops; 955 cpu_md_ops.md_tlb_io_ops = &e500_tlb_io_ops; 956 957 if (__predict_false(memmapped < memsize)) { 958 /* 959 * Let's see how many TLB entries are needed to map memory. 960 */ 961 u_int slotmask = e500_tlbmemmap(0, memsize, tlb1); 962 963 /* 964 * To map main memory into the TLB, we need to flush any 965 * existing entries from the TLB that overlap the virtual 966 * address space needed to map physical memory. That may 967 * include the entries for the pages currently used by the 968 * stack or that we are executing. So to avoid problems, we 969 * are going to temporarily map the kernel and stack into AS 1, 970 * switch to it, and clear out the TLB entries from AS 0, 971 * install the new TLB entries to map memory, and then switch 972 * back to AS 0 and free the temp entry used for AS1. 973 */ 974 u_int b = __builtin_clz(endkernel); 975 976 /* 977 * If the kernel doesn't end on a clean power of 2, we need 978 * to round the size up (by decrementing the number of leading 979 * zero bits). If the size isn't a power of 4KB, decrement 980 * again to make it one. 981 */ 982 if (endkernel & (endkernel - 1)) 983 b--; 984 if ((b & 1) == 0) 985 b--; 986 987 /* 988 * Create a TLB1 mapping for the kernel in AS1. 989 */ 990 const u_int kslot = e500_alloc_tlb1_entry(); 991 struct e500_xtlb * const kxtlb = &tlb1->tlb1_entries[kslot]; 992 kxtlb->e_tlb.tlb_va = 0; 993 kxtlb->e_tlb.tlb_size = 1UL << (31 - b); 994 kxtlb->e_tlb.tlb_pte = PTE_M|PTE_xR|PTE_xW|PTE_xX; 995 kxtlb->e_tlb.tlb_asid = KERNEL_PID; 996 997 kxtlb->e_hwtlb = tlb_to_hwtlb(kxtlb->e_tlb); 998 kxtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTIN(kslot, MAS0_ESEL); 999 kxtlb->e_hwtlb.hwtlb_mas1 |= MAS1_TS; 1000 hwtlb_write(kxtlb->e_hwtlb, true); 1001 1002 /* 1003 * Now that we have a TLB mapping in AS1 for the kernel and its 1004 * stack, we switch to AS1 to cleanup the TLB mappings for TLB0. 1005 */ 1006 const register_t saved_msr = mfmsr(); 1007 mtmsr(saved_msr | PSL_DS | PSL_IS); 1008 __asm volatile("isync"); 1009 1010 /* 1011 *** Invalidate all the TLB0 entries. 1012 */ 1013 e500_tlb_invalidate_all(); 1014 1015 /* 1016 *** Now let's see if we have any entries in TLB1 that would 1017 *** overlap the ones we are about to install. If so, nuke 'em. 1018 */ 1019 for (u_int i = 0; i < tlb1->tlb1_numentries; i++) { 1020 struct e500_xtlb * const xtlb = &tlb1->tlb1_entries[i]; 1021 struct e500_hwtlb * const hwtlb = &xtlb->e_hwtlb; 1022 if ((hwtlb->hwtlb_mas1 & (MAS1_V|MAS1_TS)) == MAS1_V 1023 && (hwtlb->hwtlb_mas2 & MAS2_EPN) < memsize) { 1024 e500_free_tlb1_entry(xtlb, i, false); 1025 } 1026 } 1027 1028 /* 1029 *** Now we can add the TLB entries that will map physical 1030 *** memory. If bit 0 [MSB] in slotmask is set, then tlb 1031 *** entry 0 contains a mapping for physical memory... 1032 */ 1033 struct e500_xtlb *entries = tlb1->tlb1_entries; 1034 while (slotmask != 0) { 1035 const u_int slot = __builtin_clz(slotmask); 1036 hwtlb_write(entries[slot].e_hwtlb, false); 1037 entries += slot + 1; 1038 slotmask <<= slot + 1; 1039 } 1040 1041 /* 1042 *** Synchronize the TLB and the instruction stream. 1043 */ 1044 __asm volatile("tlbsync"); 1045 __asm volatile("isync"); 1046 1047 /* 1048 *** Switch back to AS 0. 1049 */ 1050 mtmsr(saved_msr); 1051 __asm volatile("isync"); 1052 1053 /* 1054 * Free the temporary TLB1 entry. 1055 */ 1056 e500_free_tlb1_entry(kxtlb, kslot, true); 1057 } 1058 1059 /* 1060 * Finally set the MAS4 defaults. 1061 */ 1062 mtspr(SPR_MAS4, MAS4_TSIZED_4KB | MAS4_MD); 1063 1064 /* 1065 * Invalidate all the TLB0 entries. 1066 */ 1067 e500_tlb_invalidate_all(); 1068 } 1069 1070 void 1071 e500_tlb_minimize(vaddr_t endkernel) 1072 { 1073 #ifdef PMAP_MINIMALTLB 1074 struct e500_tlb1 * const tlb1 = &e500_tlb1; 1075 extern uint32_t _fdata[]; 1076 1077 u_int slot; 1078 1079 paddr_t boot_page = cpu_read_4(GUR_BPTR); 1080 if (boot_page & BPTR_EN) { 1081 /* 1082 * shift it to an address 1083 */ 1084 boot_page = (boot_page & BPTR_BOOT_PAGE) << PAGE_SHIFT; 1085 pmap_kvptefill(boot_page, boot_page + NBPG, 1086 PTE_M | PTE_xR | PTE_xW | PTE_xX); 1087 } 1088 1089 1090 KASSERT(endkernel - (uintptr_t)_fdata < 0x400000); 1091 KASSERT((uintptr_t)_fdata == 0x400000); 1092 1093 struct e500_xtlb *xtlb = e500_tlb_lookup_xtlb(endkernel, &slot); 1094 1095 KASSERT(xtlb == e500_tlb_lookup_xtlb2(0, endkernel)); 1096 const u_int tmp_slot = e500_alloc_tlb1_entry(); 1097 KASSERT(tmp_slot != (u_int) -1); 1098 1099 struct e500_xtlb * const tmp_xtlb = &tlb1->tlb1_entries[tmp_slot]; 1100 tmp_xtlb->e_tlb = xtlb->e_tlb; 1101 tmp_xtlb->e_hwtlb = tlb_to_hwtlb(tmp_xtlb->e_tlb); 1102 tmp_xtlb->e_hwtlb.hwtlb_mas1 |= MAS1_TS; 1103 KASSERT((tmp_xtlb->e_hwtlb.hwtlb_mas0 & MAS0_TLBSEL) == MAS0_TLBSEL_TLB1); 1104 tmp_xtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTIN(tmp_slot, MAS0_ESEL); 1105 hwtlb_write(tmp_xtlb->e_hwtlb, true); 1106 1107 const u_int text_slot = e500_alloc_tlb1_entry(); 1108 KASSERT(text_slot != (u_int)-1); 1109 struct e500_xtlb * const text_xtlb = &tlb1->tlb1_entries[text_slot]; 1110 text_xtlb->e_tlb.tlb_va = 0; 1111 text_xtlb->e_tlb.tlb_size = 0x400000; 1112 text_xtlb->e_tlb.tlb_pte = PTE_M | PTE_xR | PTE_xX | text_xtlb->e_tlb.tlb_va; 1113 text_xtlb->e_tlb.tlb_asid = 0; 1114 text_xtlb->e_hwtlb = tlb_to_hwtlb(text_xtlb->e_tlb); 1115 KASSERT((text_xtlb->e_hwtlb.hwtlb_mas0 & MAS0_TLBSEL) == MAS0_TLBSEL_TLB1); 1116 text_xtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTIN(text_slot, MAS0_ESEL); 1117 1118 const u_int data_slot = e500_alloc_tlb1_entry(); 1119 KASSERT(data_slot != (u_int)-1); 1120 struct e500_xtlb * const data_xtlb = &tlb1->tlb1_entries[data_slot]; 1121 data_xtlb->e_tlb.tlb_va = 0x400000; 1122 data_xtlb->e_tlb.tlb_size = 0x400000; 1123 data_xtlb->e_tlb.tlb_pte = PTE_M | PTE_xR | PTE_xW | data_xtlb->e_tlb.tlb_va; 1124 data_xtlb->e_tlb.tlb_asid = 0; 1125 data_xtlb->e_hwtlb = tlb_to_hwtlb(data_xtlb->e_tlb); 1126 KASSERT((data_xtlb->e_hwtlb.hwtlb_mas0 & MAS0_TLBSEL) == MAS0_TLBSEL_TLB1); 1127 data_xtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTIN(data_slot, MAS0_ESEL); 1128 1129 const register_t msr = mfmsr(); 1130 const register_t ts_msr = (msr | PSL_DS | PSL_IS) & ~PSL_EE; 1131 1132 __asm __volatile( 1133 "mtmsr %[ts_msr]" "\n\t" 1134 "sync" "\n\t" 1135 "isync" 1136 :: [ts_msr] "r" (ts_msr)); 1137 1138 #if 0 1139 hwtlb_write(text_xtlb->e_hwtlb, false); 1140 hwtlb_write(data_xtlb->e_hwtlb, false); 1141 e500_free_tlb1_entry(xtlb, slot, true); 1142 #endif 1143 1144 __asm __volatile( 1145 "mtmsr %[msr]" "\n\t" 1146 "sync" "\n\t" 1147 "isync" 1148 :: [msr] "r" (msr)); 1149 1150 e500_free_tlb1_entry(tmp_xtlb, tmp_slot, true); 1151 #endif /* PMAP_MINIMALTLB */ 1152 } 1153