1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD AND 4-Clause-BSD 3 * 4 * Copyright (c) 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Matt Thomas <matt@3am-software.com> of Allegro Networks, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 /*- 32 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 33 * Copyright (C) 1995, 1996 TooLs GmbH. 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 3. All advertising materials mentioning features or use of this software 45 * must display the following acknowledgement: 46 * This product includes software developed by TooLs GmbH. 47 * 4. The name of TooLs GmbH may not be used to endorse or promote products 48 * derived from this software without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 51 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 52 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 53 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 54 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 55 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 56 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 57 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 58 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 59 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 60 * 61 * $NetBSD: pmap.c,v 1.28 2000/03/26 20:42:36 kleink Exp $ 62 */ 63 /*- 64 * Copyright (C) 2001 Benno Rice. 65 * All rights reserved. 66 * 67 * Redistribution and use in source and binary forms, with or without 68 * modification, are permitted provided that the following conditions 69 * are met: 70 * 1. Redistributions of source code must retain the above copyright 71 * notice, this list of conditions and the following disclaimer. 72 * 2. Redistributions in binary form must reproduce the above copyright 73 * notice, this list of conditions and the following disclaimer in the 74 * documentation and/or other materials provided with the distribution. 75 * 76 * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR 77 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 78 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 79 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 80 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 81 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 82 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 83 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 84 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 85 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 86 */ 87 88 #include <sys/cdefs.h> 89 __FBSDID("$FreeBSD$"); 90 91 /* 92 * Native 64-bit page table operations for running without a hypervisor. 93 */ 94 95 #include <sys/param.h> 96 #include <sys/kernel.h> 97 #include <sys/ktr.h> 98 #include <sys/lock.h> 99 #include <sys/mutex.h> 100 #include <sys/proc.h> 101 #include <sys/sched.h> 102 #include <sys/sysctl.h> 103 #include <sys/systm.h> 104 #include <sys/rwlock.h> 105 #include <sys/endian.h> 106 107 #include <sys/kdb.h> 108 109 #include <vm/vm.h> 110 #include <vm/vm_param.h> 111 #include <vm/vm_kern.h> 112 #include <vm/vm_page.h> 113 #include <vm/vm_map.h> 114 #include <vm/vm_object.h> 115 #include <vm/vm_extern.h> 116 #include <vm/vm_pageout.h> 117 118 #include <machine/cpu.h> 119 #include <machine/hid.h> 120 #include <machine/md_var.h> 121 #include <machine/mmuvar.h> 122 123 #include "mmu_oea64.h" 124 #include "mmu_if.h" 125 #include "moea64_if.h" 126 127 #define PTESYNC() __asm __volatile("ptesync"); 128 #define TLBSYNC() __asm __volatile("tlbsync; ptesync"); 129 #define SYNC() __asm __volatile("sync"); 130 #define EIEIO() __asm __volatile("eieio"); 131 132 #define VSID_HASH_MASK 0x0000007fffffffffULL 133 134 /* POWER9 only permits a 64k partition table size. */ 135 #define PART_SIZE 0x10000 136 137 static bool moea64_crop_tlbie; 138 static bool moea64_need_lock; 139 140 static __inline void 141 TLBIE(uint64_t vpn) { 142 #ifndef __powerpc64__ 143 register_t vpn_hi, vpn_lo; 144 register_t msr; 145 register_t scratch, intr; 146 #endif 147 148 static volatile u_int tlbie_lock = 0; 149 150 vpn <<= ADDR_PIDX_SHFT; 151 152 /* Hobo spinlock: we need stronger guarantees than mutexes provide */ 153 if (moea64_need_lock) { 154 while (!atomic_cmpset_int(&tlbie_lock, 0, 1)); 155 isync(); /* Flush instruction queue once lock acquired */ 156 } 157 158 if (moea64_crop_tlbie) 159 vpn &= ~(0xffffULL << 48); 160 161 #ifdef __powerpc64__ 162 /* 163 * Explicitly clobber r0. The tlbie instruction has two forms: an old 164 * one used by PowerISA 2.03 and prior, and a newer one used by PowerISA 165 * 2.06 (maybe 2.05?) and later. We need to support both, and it just 166 * so happens that since we use 4k pages we can simply zero out r0, and 167 * clobber it, and the assembler will interpret the single-operand form 168 * of tlbie as having RB set, and everything else as 0. The RS operand 169 * in the newer form is in the same position as the L(page size) bit of 170 * the old form, so a slong as RS is 0, we're good on both sides. 171 */ 172 __asm __volatile("li 0, 0 \n tlbie %0" :: "r"(vpn) : "r0", "memory"); 173 __asm __volatile("eieio; tlbsync; ptesync" ::: "memory"); 174 #else 175 vpn_hi = (uint32_t)(vpn >> 32); 176 vpn_lo = (uint32_t)vpn; 177 178 intr = intr_disable(); 179 __asm __volatile("\ 180 mfmsr %0; \ 181 mr %1, %0; \ 182 insrdi %1,%5,1,0; \ 183 mtmsrd %1; isync; \ 184 \ 185 sld %1,%2,%4; \ 186 or %1,%1,%3; \ 187 tlbie %1; \ 188 \ 189 mtmsrd %0; isync; \ 190 eieio; \ 191 tlbsync; \ 192 ptesync;" 193 : "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32), "r"(1) 194 : "memory"); 195 intr_restore(intr); 196 #endif 197 198 /* No barriers or special ops -- taken care of by ptesync above */ 199 if (moea64_need_lock) 200 tlbie_lock = 0; 201 } 202 203 #define DISABLE_TRANS(msr) msr = mfmsr(); mtmsr(msr & ~PSL_DR) 204 #define ENABLE_TRANS(msr) mtmsr(msr) 205 206 /* 207 * PTEG data. 208 */ 209 static volatile struct lpte *moea64_pteg_table; 210 static struct rwlock moea64_eviction_lock; 211 212 static volatile struct pate *moea64_part_table; 213 214 /* 215 * PTE calls. 216 */ 217 static int moea64_pte_insert_native(mmu_t, struct pvo_entry *); 218 static int64_t moea64_pte_synch_native(mmu_t, struct pvo_entry *); 219 static int64_t moea64_pte_clear_native(mmu_t, struct pvo_entry *, uint64_t); 220 static int64_t moea64_pte_replace_native(mmu_t, struct pvo_entry *, int); 221 static int64_t moea64_pte_unset_native(mmu_t mmu, struct pvo_entry *); 222 223 /* 224 * Utility routines. 225 */ 226 static void moea64_bootstrap_native(mmu_t mmup, 227 vm_offset_t kernelstart, vm_offset_t kernelend); 228 static void moea64_cpu_bootstrap_native(mmu_t, int ap); 229 static void tlbia(void); 230 231 static mmu_method_t moea64_native_methods[] = { 232 /* Internal interfaces */ 233 MMUMETHOD(mmu_bootstrap, moea64_bootstrap_native), 234 MMUMETHOD(mmu_cpu_bootstrap, moea64_cpu_bootstrap_native), 235 236 MMUMETHOD(moea64_pte_synch, moea64_pte_synch_native), 237 MMUMETHOD(moea64_pte_clear, moea64_pte_clear_native), 238 MMUMETHOD(moea64_pte_unset, moea64_pte_unset_native), 239 MMUMETHOD(moea64_pte_replace, moea64_pte_replace_native), 240 MMUMETHOD(moea64_pte_insert, moea64_pte_insert_native), 241 242 { 0, 0 } 243 }; 244 245 MMU_DEF_INHERIT(oea64_mmu_native, MMU_TYPE_G5, moea64_native_methods, 246 0, oea64_mmu); 247 248 static int64_t 249 moea64_pte_synch_native(mmu_t mmu, struct pvo_entry *pvo) 250 { 251 volatile struct lpte *pt = moea64_pteg_table + pvo->pvo_pte.slot; 252 struct lpte properpt; 253 uint64_t ptelo; 254 255 PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED); 256 257 moea64_pte_from_pvo(pvo, &properpt); 258 259 rw_rlock(&moea64_eviction_lock); 260 if ((be64toh(pt->pte_hi) & LPTE_AVPN_MASK) != 261 (properpt.pte_hi & LPTE_AVPN_MASK)) { 262 /* Evicted */ 263 rw_runlock(&moea64_eviction_lock); 264 return (-1); 265 } 266 267 PTESYNC(); 268 ptelo = be64toh(pt->pte_lo); 269 270 rw_runlock(&moea64_eviction_lock); 271 272 return (ptelo & (LPTE_REF | LPTE_CHG)); 273 } 274 275 static int64_t 276 moea64_pte_clear_native(mmu_t mmu, struct pvo_entry *pvo, uint64_t ptebit) 277 { 278 volatile struct lpte *pt = moea64_pteg_table + pvo->pvo_pte.slot; 279 struct lpte properpt; 280 uint64_t ptelo; 281 282 PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED); 283 284 moea64_pte_from_pvo(pvo, &properpt); 285 286 rw_rlock(&moea64_eviction_lock); 287 if ((be64toh(pt->pte_hi) & LPTE_AVPN_MASK) != 288 (properpt.pte_hi & LPTE_AVPN_MASK)) { 289 /* Evicted */ 290 rw_runlock(&moea64_eviction_lock); 291 return (-1); 292 } 293 294 if (ptebit == LPTE_REF) { 295 /* See "Resetting the Reference Bit" in arch manual */ 296 PTESYNC(); 297 /* 2-step here safe: precision is not guaranteed */ 298 ptelo = be64toh(pt->pte_lo); 299 300 /* One-byte store to avoid touching the C bit */ 301 ((volatile uint8_t *)(&pt->pte_lo))[6] = 302 #if BYTE_ORDER == BIG_ENDIAN 303 ((uint8_t *)(&properpt.pte_lo))[6]; 304 #else 305 ((uint8_t *)(&properpt.pte_lo))[1]; 306 #endif 307 rw_runlock(&moea64_eviction_lock); 308 309 critical_enter(); 310 TLBIE(pvo->pvo_vpn); 311 critical_exit(); 312 } else { 313 rw_runlock(&moea64_eviction_lock); 314 ptelo = moea64_pte_unset_native(mmu, pvo); 315 moea64_pte_insert_native(mmu, pvo); 316 } 317 318 return (ptelo & (LPTE_REF | LPTE_CHG)); 319 } 320 321 static int64_t 322 moea64_pte_unset_native(mmu_t mmu, struct pvo_entry *pvo) 323 { 324 volatile struct lpte *pt = moea64_pteg_table + pvo->pvo_pte.slot; 325 struct lpte properpt; 326 uint64_t ptelo; 327 328 moea64_pte_from_pvo(pvo, &properpt); 329 330 rw_rlock(&moea64_eviction_lock); 331 if ((be64toh(pt->pte_hi & LPTE_AVPN_MASK)) != 332 (properpt.pte_hi & LPTE_AVPN_MASK)) { 333 /* Evicted */ 334 moea64_pte_overflow--; 335 rw_runlock(&moea64_eviction_lock); 336 return (-1); 337 } 338 339 /* 340 * Invalidate the pte, briefly locking it to collect RC bits. No 341 * atomics needed since this is protected against eviction by the lock. 342 */ 343 isync(); 344 critical_enter(); 345 pt->pte_hi = be64toh((pt->pte_hi & ~LPTE_VALID) | LPTE_LOCKED); 346 PTESYNC(); 347 TLBIE(pvo->pvo_vpn); 348 ptelo = be64toh(pt->pte_lo); 349 *((volatile int32_t *)(&pt->pte_hi) + 1) = 0; /* Release lock */ 350 critical_exit(); 351 rw_runlock(&moea64_eviction_lock); 352 353 /* Keep statistics */ 354 moea64_pte_valid--; 355 356 return (ptelo & (LPTE_CHG | LPTE_REF)); 357 } 358 359 static int64_t 360 moea64_pte_replace_native(mmu_t mmu, struct pvo_entry *pvo, int flags) 361 { 362 volatile struct lpte *pt = moea64_pteg_table + pvo->pvo_pte.slot; 363 struct lpte properpt; 364 int64_t ptelo; 365 366 if (flags == 0) { 367 /* Just some software bits changing. */ 368 moea64_pte_from_pvo(pvo, &properpt); 369 370 rw_rlock(&moea64_eviction_lock); 371 if ((be64toh(pt->pte_hi) & LPTE_AVPN_MASK) != 372 (properpt.pte_hi & LPTE_AVPN_MASK)) { 373 rw_runlock(&moea64_eviction_lock); 374 return (-1); 375 } 376 pt->pte_hi = htobe64(properpt.pte_hi); 377 ptelo = be64toh(pt->pte_lo); 378 rw_runlock(&moea64_eviction_lock); 379 } else { 380 /* Otherwise, need reinsertion and deletion */ 381 ptelo = moea64_pte_unset_native(mmu, pvo); 382 moea64_pte_insert_native(mmu, pvo); 383 } 384 385 return (ptelo); 386 } 387 388 static void 389 moea64_cpu_bootstrap_native(mmu_t mmup, int ap) 390 { 391 int i = 0; 392 #ifdef __powerpc64__ 393 struct slb *slb = PCPU_GET(aim.slb); 394 register_t seg0; 395 #endif 396 397 /* 398 * Initialize segment registers and MMU 399 */ 400 401 mtmsr(mfmsr() & ~PSL_DR & ~PSL_IR); 402 403 /* 404 * Install kernel SLB entries 405 */ 406 407 #ifdef __powerpc64__ 408 __asm __volatile ("slbia"); 409 __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : 410 "r"(0)); 411 412 for (i = 0; i < n_slbs; i++) { 413 if (!(slb[i].slbe & SLBE_VALID)) 414 continue; 415 416 __asm __volatile ("slbmte %0, %1" :: 417 "r"(slb[i].slbv), "r"(slb[i].slbe)); 418 } 419 #else 420 for (i = 0; i < 16; i++) 421 mtsrin(i << ADDR_SR_SHFT, kernel_pmap->pm_sr[i]); 422 #endif 423 424 /* 425 * Install page table 426 */ 427 428 if (cpu_features2 & PPC_FEATURE2_ARCH_3_00) 429 mtspr(SPR_PTCR, 430 ((uintptr_t)moea64_part_table & ~DMAP_BASE_ADDRESS) | 431 flsl((PART_SIZE >> 12) - 1)); 432 else 433 __asm __volatile ("ptesync; mtsdr1 %0; isync" 434 :: "r"(((uintptr_t)moea64_pteg_table & ~DMAP_BASE_ADDRESS) 435 | (uintptr_t)(flsl(moea64_pteg_mask >> 11)))); 436 tlbia(); 437 } 438 439 static void 440 moea64_bootstrap_native(mmu_t mmup, vm_offset_t kernelstart, 441 vm_offset_t kernelend) 442 { 443 vm_size_t size; 444 vm_offset_t off; 445 vm_paddr_t pa; 446 register_t msr; 447 448 moea64_early_bootstrap(mmup, kernelstart, kernelend); 449 450 switch (mfpvr() >> 16) { 451 case IBMPOWER9: 452 moea64_need_lock = false; 453 break; 454 case IBMPOWER4: 455 case IBMPOWER4PLUS: 456 case IBM970: 457 case IBM970FX: 458 case IBM970GX: 459 case IBM970MP: 460 moea64_crop_tlbie = true; 461 default: 462 moea64_need_lock = true; 463 } 464 /* 465 * Allocate PTEG table. 466 */ 467 468 size = moea64_pteg_count * sizeof(struct lpteg); 469 CTR2(KTR_PMAP, "moea64_bootstrap: %lu PTEGs, %lu bytes", 470 moea64_pteg_count, size); 471 rw_init(&moea64_eviction_lock, "pte eviction"); 472 473 /* 474 * We now need to allocate memory. This memory, to be allocated, 475 * has to reside in a page table. The page table we are about to 476 * allocate. We don't have BAT. So drop to data real mode for a minute 477 * as a measure of last resort. We do this a couple times. 478 */ 479 /* 480 * PTEG table must be aligned on a 256k boundary, but can be placed 481 * anywhere with that alignment on POWER ISA 3+ systems. On earlier 482 * systems, offset addition is done by the CPU with bitwise OR rather 483 * than addition, so the table must also be aligned on a boundary of 484 * its own size. Pick the larger of the two, which works on all 485 * systems. 486 */ 487 moea64_pteg_table = (struct lpte *)moea64_bootstrap_alloc(size, 488 MAX(256*1024, size)); 489 if (hw_direct_map) 490 moea64_pteg_table = 491 (struct lpte *)PHYS_TO_DMAP((vm_offset_t)moea64_pteg_table); 492 /* Allocate partition table (ISA 3.0). */ 493 if (cpu_features2 & PPC_FEATURE2_ARCH_3_00) { 494 moea64_part_table = 495 (struct pate *)moea64_bootstrap_alloc(PART_SIZE, PART_SIZE); 496 if (hw_direct_map) 497 moea64_part_table = 498 (struct pate *)PHYS_TO_DMAP((vm_offset_t)moea64_part_table); 499 } 500 DISABLE_TRANS(msr); 501 bzero(__DEVOLATILE(void *, moea64_pteg_table), moea64_pteg_count * 502 sizeof(struct lpteg)); 503 if (cpu_features2 & PPC_FEATURE2_ARCH_3_00) { 504 bzero(__DEVOLATILE(void *, moea64_part_table), PART_SIZE); 505 moea64_part_table[0].pagetab = 506 (DMAP_TO_PHYS((vm_offset_t)moea64_pteg_table)) | 507 (uintptr_t)(flsl((moea64_pteg_count - 1) >> 11)); 508 } 509 ENABLE_TRANS(msr); 510 511 CTR1(KTR_PMAP, "moea64_bootstrap: PTEG table at %p", moea64_pteg_table); 512 513 moea64_mid_bootstrap(mmup, kernelstart, kernelend); 514 515 /* 516 * Add a mapping for the page table itself if there is no direct map. 517 */ 518 if (!hw_direct_map) { 519 size = moea64_pteg_count * sizeof(struct lpteg); 520 off = (vm_offset_t)(moea64_pteg_table); 521 DISABLE_TRANS(msr); 522 for (pa = off; pa < off + size; pa += PAGE_SIZE) 523 pmap_kenter(pa, pa); 524 ENABLE_TRANS(msr); 525 } 526 527 /* Bring up virtual memory */ 528 moea64_late_bootstrap(mmup, kernelstart, kernelend); 529 } 530 531 static void 532 tlbia(void) 533 { 534 vm_offset_t i; 535 #ifndef __powerpc64__ 536 register_t msr, scratch; 537 #endif 538 539 i = 0xc00; /* IS = 11 */ 540 switch (mfpvr() >> 16) { 541 case IBM970: 542 case IBM970FX: 543 case IBM970MP: 544 case IBM970GX: 545 case IBMPOWER4: 546 case IBMPOWER4PLUS: 547 case IBMPOWER5: 548 case IBMPOWER5PLUS: 549 i = 0; /* IS not supported */ 550 break; 551 } 552 553 TLBSYNC(); 554 555 for (; i < 0x400000; i += 0x00001000) { 556 #ifdef __powerpc64__ 557 __asm __volatile("tlbiel %0" :: "r"(i)); 558 #else 559 __asm __volatile("\ 560 mfmsr %0; \ 561 mr %1, %0; \ 562 insrdi %1,%3,1,0; \ 563 mtmsrd %1; \ 564 isync; \ 565 \ 566 tlbiel %2; \ 567 \ 568 mtmsrd %0; \ 569 isync;" 570 : "=r"(msr), "=r"(scratch) : "r"(i), "r"(1)); 571 #endif 572 } 573 574 EIEIO(); 575 TLBSYNC(); 576 } 577 578 static int 579 atomic_pte_lock(volatile struct lpte *pte, uint64_t bitmask, uint64_t *oldhi) 580 { 581 int ret; 582 uint32_t oldhihalf; 583 584 /* 585 * Note: in principle, if just the locked bit were set here, we 586 * could avoid needing the eviction lock. However, eviction occurs 587 * so rarely that it isn't worth bothering about in practice. 588 */ 589 590 __asm __volatile ( 591 "1:\tlwarx %1, 0, %3\n\t" /* load old value */ 592 "and. %0,%1,%4\n\t" /* check if any bits set */ 593 "bne 2f\n\t" /* exit if any set */ 594 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 595 "bne- 1b\n\t" /* spin if failed */ 596 "li %0, 1\n\t" /* success - retval = 1 */ 597 "b 3f\n\t" /* we've succeeded */ 598 "2:\n\t" 599 "stwcx. %1, 0, %3\n\t" /* clear reservation (74xx) */ 600 "li %0, 0\n\t" /* failure - retval = 0 */ 601 "3:\n\t" 602 : "=&r" (ret), "=&r"(oldhihalf), "=m" (pte->pte_hi) 603 : "r" ((volatile char *)&pte->pte_hi + 4), 604 "r" ((uint32_t)bitmask), "r" ((uint32_t)LPTE_LOCKED), 605 "m" (pte->pte_hi) 606 : "cr0", "cr1", "cr2", "memory"); 607 608 *oldhi = (pte->pte_hi & 0xffffffff00000000ULL) | oldhihalf; 609 610 return (ret); 611 } 612 613 static uintptr_t 614 moea64_insert_to_pteg_native(struct lpte *pvo_pt, uintptr_t slotbase, 615 uint64_t mask) 616 { 617 volatile struct lpte *pt; 618 uint64_t oldptehi, va; 619 uintptr_t k; 620 int i, j; 621 622 /* Start at a random slot */ 623 i = mftb() % 8; 624 for (j = 0; j < 8; j++) { 625 k = slotbase + (i + j) % 8; 626 pt = &moea64_pteg_table[k]; 627 /* Invalidate and seize lock only if no bits in mask set */ 628 if (atomic_pte_lock(pt, mask, &oldptehi)) /* Lock obtained */ 629 break; 630 } 631 632 if (j == 8) 633 return (-1); 634 635 if (oldptehi & LPTE_VALID) { 636 KASSERT(!(oldptehi & LPTE_WIRED), ("Unmapped wired entry")); 637 /* 638 * Need to invalidate old entry completely: see 639 * "Modifying a Page Table Entry". Need to reconstruct 640 * the virtual address for the outgoing entry to do that. 641 */ 642 if (oldptehi & LPTE_BIG) 643 va = oldptehi >> moea64_large_page_shift; 644 else 645 va = oldptehi >> ADDR_PIDX_SHFT; 646 if (oldptehi & LPTE_HID) 647 va = (((k >> 3) ^ moea64_pteg_mask) ^ va) & 648 VSID_HASH_MASK; 649 else 650 va = ((k >> 3) ^ va) & VSID_HASH_MASK; 651 va |= (oldptehi & LPTE_AVPN_MASK) << 652 (ADDR_API_SHFT64 - ADDR_PIDX_SHFT); 653 PTESYNC(); 654 TLBIE(va); 655 moea64_pte_valid--; 656 moea64_pte_overflow++; 657 } 658 659 /* 660 * Update the PTE as per "Adding a Page Table Entry". Lock is released 661 * by setting the high doubleworld. 662 */ 663 pt->pte_lo = htobe64(pvo_pt->pte_lo); 664 EIEIO(); 665 pt->pte_hi = htobe64(pvo_pt->pte_hi); 666 PTESYNC(); 667 668 /* Keep statistics */ 669 moea64_pte_valid++; 670 671 return (k); 672 } 673 674 static int 675 moea64_pte_insert_native(mmu_t mmu, struct pvo_entry *pvo) 676 { 677 struct lpte insertpt; 678 uintptr_t slot; 679 680 /* Initialize PTE */ 681 moea64_pte_from_pvo(pvo, &insertpt); 682 683 /* Make sure further insertion is locked out during evictions */ 684 rw_rlock(&moea64_eviction_lock); 685 686 /* 687 * First try primary hash. 688 */ 689 pvo->pvo_pte.slot &= ~7ULL; /* Base slot address */ 690 slot = moea64_insert_to_pteg_native(&insertpt, pvo->pvo_pte.slot, 691 LPTE_VALID | LPTE_WIRED | LPTE_LOCKED); 692 if (slot != -1) { 693 rw_runlock(&moea64_eviction_lock); 694 pvo->pvo_pte.slot = slot; 695 return (0); 696 } 697 698 /* 699 * Now try secondary hash. 700 */ 701 pvo->pvo_vaddr ^= PVO_HID; 702 insertpt.pte_hi ^= LPTE_HID; 703 pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3); 704 slot = moea64_insert_to_pteg_native(&insertpt, pvo->pvo_pte.slot, 705 LPTE_VALID | LPTE_WIRED | LPTE_LOCKED); 706 if (slot != -1) { 707 rw_runlock(&moea64_eviction_lock); 708 pvo->pvo_pte.slot = slot; 709 return (0); 710 } 711 712 /* 713 * Out of luck. Find a PTE to sacrifice. 714 */ 715 716 /* Lock out all insertions for a bit */ 717 if (!rw_try_upgrade(&moea64_eviction_lock)) { 718 rw_runlock(&moea64_eviction_lock); 719 rw_wlock(&moea64_eviction_lock); 720 } 721 722 slot = moea64_insert_to_pteg_native(&insertpt, pvo->pvo_pte.slot, 723 LPTE_WIRED | LPTE_LOCKED); 724 if (slot != -1) { 725 rw_wunlock(&moea64_eviction_lock); 726 pvo->pvo_pte.slot = slot; 727 return (0); 728 } 729 730 /* Try other hash table. Now we're getting desperate... */ 731 pvo->pvo_vaddr ^= PVO_HID; 732 insertpt.pte_hi ^= LPTE_HID; 733 pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3); 734 slot = moea64_insert_to_pteg_native(&insertpt, pvo->pvo_pte.slot, 735 LPTE_WIRED | LPTE_LOCKED); 736 if (slot != -1) { 737 rw_wunlock(&moea64_eviction_lock); 738 pvo->pvo_pte.slot = slot; 739 return (0); 740 } 741 742 /* No freeable slots in either PTEG? We're hosed. */ 743 rw_wunlock(&moea64_eviction_lock); 744 panic("moea64_pte_insert: overflow"); 745 return (-1); 746 } 747 748