1 /* $NetBSD: pmap.c,v 1.90 2020/07/06 10:40:21 rin Exp $ */ 2 3 /* 4 * Copyright 2001 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Eduardo Horvath and Simon Burge for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 40 * Copyright (C) 1995, 1996 TooLs GmbH. 41 * All rights reserved. 42 * 43 * Redistribution and use in source and binary forms, with or without 44 * modification, are permitted provided that the following conditions 45 * are met: 46 * 1. Redistributions of source code must retain the above copyright 47 * notice, this list of conditions and the following disclaimer. 48 * 2. Redistributions in binary form must reproduce the above copyright 49 * notice, this list of conditions and the following disclaimer in the 50 * documentation and/or other materials provided with the distribution. 51 * 3. All advertising materials mentioning features or use of this software 52 * must display the following acknowledgement: 53 * This product includes software developed by TooLs GmbH. 54 * 4. The name of TooLs GmbH may not be used to endorse or promote products 55 * derived from this software without specific prior written permission. 56 * 57 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 58 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 59 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 60 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 61 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 62 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 63 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 64 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 65 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 66 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 67 */ 68 69 #include <sys/cdefs.h> 70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.90 2020/07/06 10:40:21 rin Exp $"); 71 72 #ifdef _KERNEL_OPT 73 #include "opt_ddb.h" 74 #endif 75 76 #include <sys/param.h> 77 #include <sys/cpu.h> 78 #include <sys/device.h> 79 #include <sys/kmem.h> 80 #include <sys/pool.h> 81 #include <sys/proc.h> 82 #include <sys/queue.h> 83 #include <sys/systm.h> 84 85 #include <uvm/uvm.h> 86 87 #include <machine/powerpc.h> 88 #include <machine/tlb.h> 89 90 #include <powerpc/pcb.h> 91 92 #include <powerpc/spr.h> 93 #include <powerpc/ibm4xx/spr.h> 94 95 #include <powerpc/ibm4xx/cpu.h> 96 97 /* 98 * kernmap is an array of PTEs large enough to map in 99 * 4GB. At 16KB/page it is 256K entries or 2MB. 100 */ 101 #define KERNMAP_SIZE ((0xffffffffU/PAGE_SIZE)+1) 102 void *kernmap; 103 104 #define MINCTX 2 105 #define NUMCTX 256 106 107 volatile struct pmap *ctxbusy[NUMCTX]; 108 109 #define TLBF_USED 0x1 110 #define TLBF_REF 0x2 111 #define TLBF_LOCKED 0x4 112 #define TLB_LOCKED(i) (tlb_info[(i)].ti_flags & TLBF_LOCKED) 113 114 typedef struct tlb_info_s { 115 char ti_flags; 116 char ti_ctx; /* TLB_PID assiciated with the entry */ 117 u_int ti_va; 118 } tlb_info_t; 119 120 volatile tlb_info_t tlb_info[NTLB]; 121 /* We'll use a modified FIFO replacement policy cause it's cheap */ 122 volatile int tlbnext; 123 124 static int tlb_nreserved = 0; 125 static int pmap_bootstrap_done = 0; 126 127 /* Event counters */ 128 struct evcnt tlbmiss_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 129 NULL, "cpu", "tlbmiss"); 130 struct evcnt tlbflush_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 131 NULL, "cpu", "tlbflush"); 132 struct evcnt tlbenter_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 133 NULL, "cpu", "tlbenter"); 134 EVCNT_ATTACH_STATIC(tlbmiss_ev); 135 EVCNT_ATTACH_STATIC(tlbflush_ev); 136 EVCNT_ATTACH_STATIC(tlbenter_ev); 137 138 struct pmap kernel_pmap_; 139 struct pmap *const kernel_pmap_ptr = &kernel_pmap_; 140 141 static int npgs; 142 static u_int nextavail; 143 #ifndef MSGBUFADDR 144 extern paddr_t msgbuf_paddr; 145 #endif 146 147 static struct mem_region *mem, *avail; 148 149 /* 150 * This is a cache of referenced/modified bits. 151 * Bits herein are shifted by ATTRSHFT. 152 */ 153 static char *pmap_attrib; 154 155 #define PV_WIRED 0x1 156 #define PV_WIRE(pv) ((pv)->pv_va |= PV_WIRED) 157 #define PV_UNWIRE(pv) ((pv)->pv_va &= ~PV_WIRED) 158 #define PV_ISWIRED(pv) ((pv)->pv_va & PV_WIRED) 159 #define PV_CMPVA(va,pv) (!(((pv)->pv_va ^ (va)) & (~PV_WIRED))) 160 161 struct pv_entry { 162 struct pv_entry *pv_next; /* Linked list of mappings */ 163 struct pmap *pv_pm; 164 vaddr_t pv_va; /* virtual address of mapping */ 165 }; 166 167 /* Each index corresponds to TLB_SIZE_* value. */ 168 static size_t tlbsize[] = { 169 1024, /* TLB_SIZE_1K */ 170 4096, /* TLB_SIZE_4K */ 171 16384, /* TLB_SIZE_16K */ 172 65536, /* TLB_SIZE_64K */ 173 262144, /* TLB_SIZE_256K */ 174 1048576, /* TLB_SIZE_1M */ 175 4194304, /* TLB_SIZE_4M */ 176 16777216, /* TLB_SIZE_16M */ 177 }; 178 179 struct pv_entry *pv_table; 180 static struct pool pv_pool; 181 182 static int pmap_initialized; 183 184 static int ctx_flush(int); 185 186 struct pv_entry *pa_to_pv(paddr_t); 187 static inline char *pa_to_attr(paddr_t); 188 189 static inline volatile u_int *pte_find(struct pmap *, vaddr_t); 190 static inline int pte_enter(struct pmap *, vaddr_t, u_int); 191 192 static inline int pmap_enter_pv(struct pmap *, vaddr_t, paddr_t, int); 193 static void pmap_remove_pv(struct pmap *, vaddr_t, paddr_t); 194 195 static int ppc4xx_tlb_size_mask(size_t, int *, int *); 196 197 198 struct pv_entry * 199 pa_to_pv(paddr_t pa) 200 { 201 uvm_physseg_t bank; 202 psize_t pg; 203 204 bank = uvm_physseg_find(atop(pa), &pg); 205 if (bank == UVM_PHYSSEG_TYPE_INVALID) 206 return NULL; 207 return &uvm_physseg_get_pmseg(bank)->pvent[pg]; 208 } 209 210 static inline char * 211 pa_to_attr(paddr_t pa) 212 { 213 uvm_physseg_t bank; 214 psize_t pg; 215 216 bank = uvm_physseg_find(atop(pa), &pg); 217 if (bank == UVM_PHYSSEG_TYPE_INVALID) 218 return NULL; 219 return &uvm_physseg_get_pmseg(bank)->attrs[pg]; 220 } 221 222 /* 223 * Insert PTE into page table. 224 */ 225 int 226 pte_enter(struct pmap *pm, vaddr_t va, u_int pte) 227 { 228 int seg = STIDX(va); 229 int ptn = PTIDX(va); 230 u_int oldpte; 231 232 if (!pm->pm_ptbl[seg]) { 233 /* Don't allocate a page to clear a non-existent mapping. */ 234 if (!pte) 235 return (0); 236 /* Allocate a page XXXX this will sleep! */ 237 pm->pm_ptbl[seg] = 238 (uint *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 239 UVM_KMF_WIRED | UVM_KMF_ZERO); 240 } 241 oldpte = pm->pm_ptbl[seg][ptn]; 242 pm->pm_ptbl[seg][ptn] = pte; 243 244 /* Flush entry. */ 245 ppc4xx_tlb_flush(va, pm->pm_ctx); 246 if (oldpte != pte) { 247 if (pte == 0) 248 pm->pm_stats.resident_count--; 249 else 250 pm->pm_stats.resident_count++; 251 } 252 return (1); 253 } 254 255 /* 256 * Get a pointer to a PTE in a page table. 257 */ 258 volatile u_int * 259 pte_find(struct pmap *pm, vaddr_t va) 260 { 261 int seg = STIDX(va); 262 int ptn = PTIDX(va); 263 264 if (pm->pm_ptbl[seg]) 265 return (&pm->pm_ptbl[seg][ptn]); 266 267 return (NULL); 268 } 269 270 /* 271 * This is called during initppc, before the system is really initialized. 272 */ 273 void 274 pmap_bootstrap(u_int kernelstart, u_int kernelend) 275 { 276 struct mem_region *mp, *mp1; 277 int cnt, i; 278 u_int s, e, sz; 279 280 tlbnext = tlb_nreserved; 281 282 /* 283 * Allocate the kernel page table at the end of 284 * kernel space so it's in the locked TTE. 285 */ 286 kernmap = (void *)kernelend; 287 288 /* 289 * Initialize kernel page table. 290 */ 291 for (i = 0; i < STSZ; i++) { 292 pmap_kernel()->pm_ptbl[i] = 0; 293 } 294 ctxbusy[0] = ctxbusy[1] = pmap_kernel(); 295 296 /* 297 * Announce page-size to the VM-system 298 */ 299 uvmexp.pagesize = NBPG; 300 uvm_md_init(); 301 302 /* 303 * Get memory. 304 */ 305 mem_regions(&mem, &avail); 306 for (mp = mem; mp->size; mp++) { 307 physmem += btoc(mp->size); 308 printf("+%lx,",mp->size); 309 } 310 printf("\n"); 311 ppc4xx_tlb_init(); 312 /* 313 * Count the number of available entries. 314 */ 315 for (cnt = 0, mp = avail; mp->size; mp++) 316 cnt++; 317 318 /* 319 * Page align all regions. 320 * Non-page aligned memory isn't very interesting to us. 321 * Also, sort the entries for ascending addresses. 322 */ 323 kernelstart &= ~PGOFSET; 324 kernelend = (kernelend + PGOFSET) & ~PGOFSET; 325 for (mp = avail; mp->size; mp++) { 326 s = mp->start; 327 e = mp->start + mp->size; 328 printf("%08x-%08x -> ",s,e); 329 /* 330 * Check whether this region holds all of the kernel. 331 */ 332 if (s < kernelstart && e > kernelend) { 333 avail[cnt].start = kernelend; 334 avail[cnt++].size = e - kernelend; 335 e = kernelstart; 336 } 337 /* 338 * Look whether this regions starts within the kernel. 339 */ 340 if (s >= kernelstart && s < kernelend) { 341 if (e <= kernelend) 342 goto empty; 343 s = kernelend; 344 } 345 /* 346 * Now look whether this region ends within the kernel. 347 */ 348 if (e > kernelstart && e <= kernelend) { 349 if (s >= kernelstart) 350 goto empty; 351 e = kernelstart; 352 } 353 /* 354 * Now page align the start and size of the region. 355 */ 356 s = round_page(s); 357 e = trunc_page(e); 358 if (e < s) 359 e = s; 360 sz = e - s; 361 printf("%08x-%08x = %x\n",s,e,sz); 362 /* 363 * Check whether some memory is left here. 364 */ 365 if (sz == 0) { 366 empty: 367 memmove(mp, mp + 1, 368 (cnt - (mp - avail)) * sizeof *mp); 369 cnt--; 370 mp--; 371 continue; 372 } 373 /* 374 * Do an insertion sort. 375 */ 376 npgs += btoc(sz); 377 for (mp1 = avail; mp1 < mp; mp1++) 378 if (s < mp1->start) 379 break; 380 if (mp1 < mp) { 381 memmove(mp1 + 1, mp1, (char *)mp - (char *)mp1); 382 mp1->start = s; 383 mp1->size = sz; 384 } else { 385 mp->start = s; 386 mp->size = sz; 387 } 388 } 389 390 /* 391 * We cannot do pmap_steal_memory here, 392 * since we don't run with translation enabled yet. 393 */ 394 #ifndef MSGBUFADDR 395 /* 396 * allow for msgbuf 397 */ 398 sz = round_page(MSGBUFSIZE); 399 mp = NULL; 400 for (mp1 = avail; mp1->size; mp1++) 401 if (mp1->size >= sz) 402 mp = mp1; 403 if (mp == NULL) 404 panic("not enough memory?"); 405 406 npgs -= btoc(sz); 407 msgbuf_paddr = mp->start + mp->size - sz; 408 mp->size -= sz; 409 if (mp->size <= 0) 410 memmove(mp, mp + 1, (cnt - (mp - avail)) * sizeof *mp); 411 #endif 412 413 for (mp = avail; mp->size; mp++) 414 uvm_page_physload(atop(mp->start), atop(mp->start + mp->size), 415 atop(mp->start), atop(mp->start + mp->size), 416 VM_FREELIST_DEFAULT); 417 418 /* 419 * Initialize kernel pmap and hardware. 420 */ 421 /* Setup TLB pid allocator so it knows we alreadu using PID 1 */ 422 pmap_kernel()->pm_ctx = KERNEL_PID; 423 nextavail = avail->start; 424 425 pmap_bootstrap_done = 1; 426 } 427 428 /* 429 * Restrict given range to physical memory 430 * 431 * (Used by /dev/mem) 432 */ 433 void 434 pmap_real_memory(paddr_t *start, psize_t *size) 435 { 436 struct mem_region *mp; 437 438 for (mp = mem; mp->size; mp++) { 439 if (*start + *size > mp->start && 440 *start < mp->start + mp->size) { 441 if (*start < mp->start) { 442 *size -= mp->start - *start; 443 *start = mp->start; 444 } 445 if (*start + *size > mp->start + mp->size) 446 *size = mp->start + mp->size - *start; 447 return; 448 } 449 } 450 *size = 0; 451 } 452 453 /* 454 * Initialize anything else for pmap handling. 455 * Called during vm_init(). 456 */ 457 void 458 pmap_init(void) 459 { 460 struct pv_entry *pv; 461 vsize_t sz; 462 vaddr_t addr; 463 int i, s; 464 int bank; 465 char *attr; 466 467 sz = (vsize_t)((sizeof(struct pv_entry) + 1) * npgs); 468 sz = round_page(sz); 469 addr = uvm_km_alloc(kernel_map, sz, 0, UVM_KMF_WIRED | UVM_KMF_ZERO); 470 s = splvm(); 471 pv = pv_table = (struct pv_entry *)addr; 472 for (i = npgs; --i >= 0;) 473 pv++->pv_pm = NULL; 474 pmap_attrib = (char *)pv; 475 memset(pv, 0, npgs); 476 477 pv = pv_table; 478 attr = pmap_attrib; 479 for (bank = uvm_physseg_get_first(); 480 uvm_physseg_valid_p(bank); 481 bank = uvm_physseg_get_next(bank)) { 482 sz = uvm_physseg_get_end(bank) - uvm_physseg_get_start(bank); 483 uvm_physseg_get_pmseg(bank)->pvent = pv; 484 uvm_physseg_get_pmseg(bank)->attrs = attr; 485 pv += sz; 486 attr += sz; 487 } 488 489 pmap_initialized = 1; 490 splx(s); 491 492 /* Setup a pool for additional pvlist structures */ 493 pool_init(&pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pv_entry", NULL, 494 IPL_VM); 495 } 496 497 /* 498 * How much virtual space is available to the kernel? 499 */ 500 void 501 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 502 { 503 504 #if 0 505 /* 506 * Reserve one segment for kernel virtual memory 507 */ 508 *start = (vaddr_t)(KERNEL_SR << ADDR_SR_SHFT); 509 *end = *start + SEGMENT_LENGTH; 510 #else 511 *start = (vaddr_t) VM_MIN_KERNEL_ADDRESS; 512 *end = (vaddr_t) VM_MAX_KERNEL_ADDRESS; 513 #endif 514 } 515 516 #ifdef PMAP_GROWKERNEL 517 /* 518 * Preallocate kernel page tables to a specified VA. 519 * This simply loops through the first TTE for each 520 * page table from the beginning of the kernel pmap, 521 * reads the entry, and if the result is 522 * zero (either invalid entry or no page table) it stores 523 * a zero there, populating page tables in the process. 524 * This is not the most efficient technique but i don't 525 * expect it to be called that often. 526 */ 527 extern struct vm_page *vm_page_alloc1(void); 528 extern void vm_page_free1(struct vm_page *); 529 530 vaddr_t kbreak = VM_MIN_KERNEL_ADDRESS; 531 532 vaddr_t 533 pmap_growkernel(vaddr_t maxkvaddr) 534 { 535 int s; 536 int seg; 537 paddr_t pg; 538 struct pmap *pm = pmap_kernel(); 539 540 s = splvm(); 541 542 /* Align with the start of a page table */ 543 for (kbreak &= ~(PTMAP-1); kbreak < maxkvaddr; 544 kbreak += PTMAP) { 545 seg = STIDX(kbreak); 546 547 if (pte_find(pm, kbreak)) 548 continue; 549 550 if (uvm.page_init_done) { 551 pg = (paddr_t)VM_PAGE_TO_PHYS(vm_page_alloc1()); 552 } else { 553 if (!uvm_page_physget(&pg)) 554 panic("pmap_growkernel: no memory"); 555 } 556 if (!pg) 557 panic("pmap_growkernel: no pages"); 558 pmap_zero_page((paddr_t)pg); 559 560 /* XXX This is based on all phymem being addressable */ 561 pm->pm_ptbl[seg] = (u_int *)pg; 562 } 563 splx(s); 564 return (kbreak); 565 } 566 567 /* 568 * vm_page_alloc1: 569 * 570 * Allocate and return a memory cell with no associated object. 571 */ 572 struct vm_page * 573 vm_page_alloc1(void) 574 { 575 struct vm_page *pg; 576 577 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE); 578 if (pg) { 579 pg->wire_count = 1; /* no mappings yet */ 580 pg->flags &= ~PG_BUSY; /* never busy */ 581 } 582 return pg; 583 } 584 585 /* 586 * vm_page_free1: 587 * 588 * Returns the given page to the free list, 589 * disassociating it with any VM object. 590 * 591 * Object and page must be locked prior to entry. 592 */ 593 void 594 vm_page_free1(struct vm_page *pg) 595 { 596 #ifdef DIAGNOSTIC 597 if (pg->flags != (PG_CLEAN|PG_FAKE)) { 598 printf("Freeing invalid page %p\n", pg); 599 printf("pa = %llx\n", (unsigned long long)VM_PAGE_TO_PHYS(pg)); 600 #ifdef DDB 601 Debugger(); 602 #endif 603 return; 604 } 605 #endif 606 pg->flags |= PG_BUSY; 607 pg->wire_count = 0; 608 uvm_pagefree(pg); 609 } 610 #endif 611 612 /* 613 * Create and return a physical map. 614 */ 615 struct pmap * 616 pmap_create(void) 617 { 618 struct pmap *pm; 619 620 pm = kmem_alloc(sizeof(*pm), KM_SLEEP); 621 memset(pm, 0, sizeof *pm); 622 pm->pm_refs = 1; 623 return pm; 624 } 625 626 /* 627 * Add a reference to the given pmap. 628 */ 629 void 630 pmap_reference(struct pmap *pm) 631 { 632 633 pm->pm_refs++; 634 } 635 636 /* 637 * Retire the given pmap from service. 638 * Should only be called if the map contains no valid mappings. 639 */ 640 void 641 pmap_destroy(struct pmap *pm) 642 { 643 int i; 644 645 if (--pm->pm_refs > 0) { 646 return; 647 } 648 KASSERT(pm->pm_stats.resident_count == 0); 649 KASSERT(pm->pm_stats.wired_count == 0); 650 for (i = 0; i < STSZ; i++) 651 if (pm->pm_ptbl[i]) { 652 uvm_km_free(kernel_map, (vaddr_t)pm->pm_ptbl[i], 653 PAGE_SIZE, UVM_KMF_WIRED); 654 pm->pm_ptbl[i] = NULL; 655 } 656 if (pm->pm_ctx) 657 ctx_free(pm); 658 kmem_free(pm, sizeof(*pm)); 659 } 660 661 /* 662 * Copy the range specified by src_addr/len 663 * from the source map to the range dst_addr/len 664 * in the destination map. 665 * 666 * This routine is only advisory and need not do anything. 667 */ 668 void 669 pmap_copy(struct pmap *dst_pmap, struct pmap *src_pmap, vaddr_t dst_addr, 670 vsize_t len, vaddr_t src_addr) 671 { 672 } 673 674 /* 675 * Require that all active physical maps contain no 676 * incorrect entries NOW. 677 */ 678 void 679 pmap_update(struct pmap *pmap) 680 { 681 } 682 683 /* 684 * Fill the given physical page with zeroes. 685 */ 686 void 687 pmap_zero_page(paddr_t pa) 688 { 689 690 #ifdef PPC_4XX_NOCACHE 691 memset((void *)pa, 0, PAGE_SIZE); 692 #else 693 int i; 694 695 for (i = PAGE_SIZE/CACHELINESIZE; i > 0; i--) { 696 __asm volatile ("dcbz 0,%0" :: "r"(pa)); 697 pa += CACHELINESIZE; 698 } 699 #endif 700 } 701 702 /* 703 * Copy the given physical source page to its destination. 704 */ 705 void 706 pmap_copy_page(paddr_t src, paddr_t dst) 707 { 708 709 memcpy((void *)dst, (void *)src, PAGE_SIZE); 710 dcache_wbinv_page(dst); 711 } 712 713 /* 714 * This returns != 0 on success. 715 */ 716 static inline int 717 pmap_enter_pv(struct pmap *pm, vaddr_t va, paddr_t pa, int flags) 718 { 719 struct pv_entry *pv, *npv = NULL; 720 int s; 721 722 if (!pmap_initialized) 723 return 0; 724 725 s = splvm(); 726 pv = pa_to_pv(pa); 727 if (!pv->pv_pm) { 728 /* 729 * No entries yet, use header as the first entry. 730 */ 731 pv->pv_va = va; 732 pv->pv_pm = pm; 733 pv->pv_next = NULL; 734 } else { 735 /* 736 * There is at least one other VA mapping this page. 737 * Place this entry after the header. 738 */ 739 npv = pool_get(&pv_pool, PR_NOWAIT); 740 if (npv == NULL) { 741 if ((flags & PMAP_CANFAIL) == 0) 742 panic("pmap_enter_pv: failed"); 743 splx(s); 744 return 0; 745 } 746 npv->pv_va = va; 747 npv->pv_pm = pm; 748 npv->pv_next = pv->pv_next; 749 pv->pv_next = npv; 750 pv = npv; 751 } 752 if (flags & PMAP_WIRED) { 753 PV_WIRE(pv); 754 pm->pm_stats.wired_count++; 755 } 756 splx(s); 757 return (1); 758 } 759 760 static void 761 pmap_remove_pv(struct pmap *pm, vaddr_t va, paddr_t pa) 762 { 763 struct pv_entry *pv, *npv; 764 765 /* 766 * Remove from the PV table. 767 */ 768 pv = pa_to_pv(pa); 769 if (!pv) 770 return; 771 772 /* 773 * If it is the first entry on the list, it is actually 774 * in the header and we must copy the following entry up 775 * to the header. Otherwise we must search the list for 776 * the entry. In either case we free the now unused entry. 777 */ 778 if (pm == pv->pv_pm && PV_CMPVA(va, pv)) { 779 if (PV_ISWIRED(pv)) { 780 pm->pm_stats.wired_count--; 781 } 782 if ((npv = pv->pv_next)) { 783 *pv = *npv; 784 pool_put(&pv_pool, npv); 785 } else 786 pv->pv_pm = NULL; 787 } else { 788 for (; (npv = pv->pv_next) != NULL; pv = npv) 789 if (pm == npv->pv_pm && PV_CMPVA(va, npv)) 790 break; 791 if (npv) { 792 pv->pv_next = npv->pv_next; 793 if (PV_ISWIRED(npv)) { 794 pm->pm_stats.wired_count--; 795 } 796 pool_put(&pv_pool, npv); 797 } 798 } 799 } 800 801 /* 802 * Insert physical page at pa into the given pmap at virtual address va. 803 */ 804 int 805 pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 806 { 807 int s; 808 u_int tte; 809 bool managed; 810 811 /* 812 * Have to remove any existing mapping first. 813 */ 814 pmap_remove(pm, va, va + PAGE_SIZE); 815 816 if (flags & PMAP_WIRED) 817 flags |= prot; 818 819 managed = uvm_pageismanaged(pa); 820 821 /* 822 * Generate TTE. 823 */ 824 tte = TTE_PA(pa); 825 /* XXXX -- need to support multiple page sizes. */ 826 tte |= TTE_SZ_16K; 827 #ifdef DIAGNOSTIC 828 if ((flags & (PMAP_NOCACHE | PME_WRITETHROUG)) == 829 (PMAP_NOCACHE | PME_WRITETHROUG)) 830 panic("pmap_enter: uncached & writethrough"); 831 #endif 832 if (flags & PMAP_NOCACHE) 833 /* Must be I/O mapping */ 834 tte |= TTE_I | TTE_G; 835 #ifdef PPC_4XX_NOCACHE 836 tte |= TTE_I; 837 #else 838 else if (flags & PME_WRITETHROUG) 839 /* Uncached and writethrough are not compatible */ 840 tte |= TTE_W; 841 #endif 842 if (pm == pmap_kernel()) 843 tte |= TTE_ZONE(ZONE_PRIV); 844 else 845 tte |= TTE_ZONE(ZONE_USER); 846 847 if (flags & VM_PROT_WRITE) 848 tte |= TTE_WR; 849 850 if (flags & VM_PROT_EXECUTE) 851 tte |= TTE_EX; 852 853 /* 854 * Now record mapping for later back-translation. 855 */ 856 if (pmap_initialized && managed) { 857 char *attr; 858 859 if (!pmap_enter_pv(pm, va, pa, flags)) { 860 /* Could not enter pv on a managed page */ 861 return 1; 862 } 863 864 /* Now set attributes. */ 865 attr = pa_to_attr(pa); 866 #ifdef DIAGNOSTIC 867 if (!attr) 868 panic("managed but no attr"); 869 #endif 870 if (flags & VM_PROT_ALL) 871 *attr |= PMAP_ATTR_REF; 872 if (flags & VM_PROT_WRITE) 873 *attr |= PMAP_ATTR_CHG; 874 } 875 876 s = splvm(); 877 878 /* Insert page into page table. */ 879 pte_enter(pm, va, tte); 880 881 /* If this is a real fault, enter it in the tlb */ 882 if (tte && ((flags & PMAP_WIRED) == 0)) { 883 int s2 = splhigh(); 884 ppc4xx_tlb_enter(pm->pm_ctx, va, tte); 885 splx(s2); 886 } 887 splx(s); 888 889 /* Flush the real memory from the instruction cache. */ 890 if ((prot & VM_PROT_EXECUTE) && (tte & TTE_I) == 0) 891 __syncicache((void *)pa, PAGE_SIZE); 892 893 return 0; 894 } 895 896 void 897 pmap_unwire(struct pmap *pm, vaddr_t va) 898 { 899 struct pv_entry *pv; 900 paddr_t pa; 901 int s; 902 903 if (!pmap_extract(pm, va, &pa)) { 904 return; 905 } 906 907 pv = pa_to_pv(pa); 908 if (!pv) 909 return; 910 911 s = splvm(); 912 while (pv != NULL) { 913 if (pm == pv->pv_pm && PV_CMPVA(va, pv)) { 914 if (PV_ISWIRED(pv)) { 915 PV_UNWIRE(pv); 916 pm->pm_stats.wired_count--; 917 } 918 break; 919 } 920 pv = pv->pv_next; 921 } 922 splx(s); 923 } 924 925 void 926 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 927 { 928 int s; 929 u_int tte; 930 struct pmap *pm = pmap_kernel(); 931 932 /* 933 * Have to remove any existing mapping first. 934 */ 935 936 /* 937 * Generate TTE. 938 * 939 * XXXX 940 * 941 * Since the kernel does not handle execution privileges properly, 942 * we will handle read and execute permissions together. 943 */ 944 tte = 0; 945 if (prot & VM_PROT_ALL) { 946 947 tte = TTE_PA(pa) | TTE_EX | TTE_ZONE(ZONE_PRIV); 948 /* XXXX -- need to support multiple page sizes. */ 949 tte |= TTE_SZ_16K; 950 #ifdef DIAGNOSTIC 951 if ((flags & (PMAP_NOCACHE | PME_WRITETHROUG)) == 952 (PMAP_NOCACHE | PME_WRITETHROUG)) 953 panic("pmap_kenter_pa: uncached & writethrough"); 954 #endif 955 if (flags & PMAP_NOCACHE) 956 /* Must be I/O mapping */ 957 tte |= TTE_I | TTE_G; 958 #ifdef PPC_4XX_NOCACHE 959 tte |= TTE_I; 960 #else 961 else if (prot & PME_WRITETHROUG) 962 /* Uncached and writethrough are not compatible */ 963 tte |= TTE_W; 964 #endif 965 if (prot & VM_PROT_WRITE) 966 tte |= TTE_WR; 967 } 968 969 s = splvm(); 970 971 /* Insert page into page table. */ 972 pte_enter(pm, va, tte); 973 splx(s); 974 } 975 976 void 977 pmap_kremove(vaddr_t va, vsize_t len) 978 { 979 980 while (len > 0) { 981 pte_enter(pmap_kernel(), va, 0); 982 va += PAGE_SIZE; 983 len -= PAGE_SIZE; 984 } 985 } 986 987 /* 988 * Remove the given range of mapping entries. 989 */ 990 void 991 pmap_remove(struct pmap *pm, vaddr_t va, vaddr_t endva) 992 { 993 int s; 994 paddr_t pa; 995 volatile u_int *ptp; 996 997 s = splvm(); 998 while (va < endva) { 999 1000 if ((ptp = pte_find(pm, va)) && (pa = *ptp)) { 1001 pa = TTE_PA(pa); 1002 pmap_remove_pv(pm, va, pa); 1003 *ptp = 0; 1004 ppc4xx_tlb_flush(va, pm->pm_ctx); 1005 pm->pm_stats.resident_count--; 1006 } 1007 va += PAGE_SIZE; 1008 } 1009 1010 splx(s); 1011 } 1012 1013 /* 1014 * Get the physical page address for the given pmap/virtual address. 1015 */ 1016 bool 1017 pmap_extract(struct pmap *pm, vaddr_t va, paddr_t *pap) 1018 { 1019 int seg = STIDX(va); 1020 int ptn = PTIDX(va); 1021 u_int pa = 0; 1022 int s; 1023 1024 s = splvm(); 1025 if (pm->pm_ptbl[seg] && (pa = pm->pm_ptbl[seg][ptn]) && pap) { 1026 *pap = TTE_PA(pa) | (va & PGOFSET); 1027 } 1028 splx(s); 1029 return (pa != 0); 1030 } 1031 1032 /* 1033 * Lower the protection on the specified range of this pmap. 1034 * 1035 * There are only two cases: either the protection is going to 0, 1036 * or it is going to read-only. 1037 */ 1038 void 1039 pmap_protect(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1040 { 1041 volatile u_int *ptp; 1042 int s, bic; 1043 1044 if ((prot & VM_PROT_READ) == 0) { 1045 pmap_remove(pm, sva, eva); 1046 return; 1047 } 1048 bic = 0; 1049 if ((prot & VM_PROT_WRITE) == 0) { 1050 bic |= TTE_WR; 1051 } 1052 if ((prot & VM_PROT_EXECUTE) == 0) { 1053 bic |= TTE_EX; 1054 } 1055 if (bic == 0) { 1056 return; 1057 } 1058 s = splvm(); 1059 while (sva < eva) { 1060 if ((ptp = pte_find(pm, sva)) != NULL) { 1061 *ptp &= ~bic; 1062 ppc4xx_tlb_flush(sva, pm->pm_ctx); 1063 } 1064 sva += PAGE_SIZE; 1065 } 1066 splx(s); 1067 } 1068 1069 bool 1070 pmap_check_attr(struct vm_page *pg, u_int mask, int clear) 1071 { 1072 paddr_t pa; 1073 char *attr; 1074 int s, rv; 1075 1076 /* 1077 * First modify bits in cache. 1078 */ 1079 pa = VM_PAGE_TO_PHYS(pg); 1080 attr = pa_to_attr(pa); 1081 if (attr == NULL) 1082 return false; 1083 1084 s = splvm(); 1085 rv = ((*attr & mask) != 0); 1086 if (clear) { 1087 *attr &= ~mask; 1088 pmap_page_protect(pg, mask == PMAP_ATTR_CHG ? VM_PROT_READ : 0); 1089 } 1090 splx(s); 1091 return rv; 1092 } 1093 1094 1095 /* 1096 * Lower the protection on the specified physical page. 1097 * 1098 * There are only two cases: either the protection is going to 0, 1099 * or it is going to read-only. 1100 */ 1101 void 1102 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 1103 { 1104 paddr_t pa = VM_PAGE_TO_PHYS(pg); 1105 vaddr_t va; 1106 struct pv_entry *pvh, *pv, *npv; 1107 struct pmap *pm; 1108 1109 pvh = pa_to_pv(pa); 1110 if (pvh == NULL) 1111 return; 1112 1113 /* Handle extra pvs which may be deleted in the operation */ 1114 for (pv = pvh->pv_next; pv; pv = npv) { 1115 npv = pv->pv_next; 1116 1117 pm = pv->pv_pm; 1118 va = pv->pv_va; 1119 pmap_protect(pm, va, va + PAGE_SIZE, prot); 1120 } 1121 /* Now check the head pv */ 1122 if (pvh->pv_pm) { 1123 pv = pvh; 1124 pm = pv->pv_pm; 1125 va = pv->pv_va; 1126 pmap_protect(pm, va, va + PAGE_SIZE, prot); 1127 } 1128 } 1129 1130 /* 1131 * Activate the address space for the specified process. If the process 1132 * is the current process, load the new MMU context. 1133 */ 1134 void 1135 pmap_activate(struct lwp *l) 1136 { 1137 #if 0 1138 struct pcb *pcb = lwp_getpcb(l); 1139 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 1140 1141 /* 1142 * XXX Normally performed in cpu_lwp_fork(). 1143 */ 1144 printf("pmap_activate(%p), pmap=%p\n",l,pmap); 1145 pcb->pcb_pm = pmap; 1146 #endif 1147 } 1148 1149 /* 1150 * Deactivate the specified process's address space. 1151 */ 1152 void 1153 pmap_deactivate(struct lwp *l) 1154 { 1155 } 1156 1157 /* 1158 * Synchronize caches corresponding to [addr, addr+len) in p. 1159 */ 1160 void 1161 pmap_procwr(struct proc *p, vaddr_t va, size_t len) 1162 { 1163 struct pmap *pm = p->p_vmspace->vm_map.pmap; 1164 int msr, ctx, opid, step; 1165 1166 step = CACHELINESIZE; 1167 1168 /* 1169 * Need to turn off IMMU and switch to user context. 1170 * (icbi uses DMMU). 1171 */ 1172 if (!(ctx = pm->pm_ctx)) { 1173 /* No context -- assign it one */ 1174 ctx_alloc(pm); 1175 ctx = pm->pm_ctx; 1176 } 1177 __asm volatile( 1178 "mfmsr %0;" 1179 "li %1, %7;" 1180 "andc %1,%0,%1;" 1181 "mtmsr %1;" 1182 "isync;" 1183 "mfpid %1;" 1184 "mtpid %2;" 1185 "isync;" 1186 "1:" 1187 "dcbst 0,%3;" 1188 "icbi 0,%3;" 1189 "add %3,%3,%5;" 1190 "addc. %4,%4,%6;" 1191 "bge 1b;" 1192 "sync;" 1193 "mtpid %1;" 1194 "mtmsr %0;" 1195 "isync;" 1196 : "=&r" (msr), "=&r" (opid) 1197 : "r" (ctx), "r" (va), "r" (len), "r" (step), "r" (-step), 1198 "K" (PSL_IR | PSL_DR)); 1199 } 1200 1201 1202 /* This has to be done in real mode !!! */ 1203 void 1204 ppc4xx_tlb_flush(vaddr_t va, int pid) 1205 { 1206 u_long i, found; 1207 u_long msr; 1208 1209 /* If there's no context then it can't be mapped. */ 1210 if (!pid) 1211 return; 1212 1213 __asm volatile( 1214 "mfpid %1;" /* Save PID */ 1215 "mfmsr %2;" /* Save MSR */ 1216 "li %0,0;" /* Now clear MSR */ 1217 "mtmsr %0;" 1218 "isync;" 1219 "mtpid %4;" /* Set PID */ 1220 "isync;" 1221 "tlbsx. %0,0,%3;" /* Search TLB */ 1222 "isync;" 1223 "mtpid %1;" /* Restore PID */ 1224 "mtmsr %2;" /* Restore MSR */ 1225 "isync;" 1226 "li %1,1;" 1227 "beq 1f;" 1228 "li %1,0;" 1229 "1:" 1230 : "=&r" (i), "=&r" (found), "=&r" (msr) 1231 : "r" (va), "r" (pid)); 1232 if (found && !TLB_LOCKED(i)) { 1233 /* Now flush translation */ 1234 __asm volatile( 1235 "tlbwe %0,%1,0;" 1236 "isync;" 1237 : : "r" (0), "r" (i)); 1238 1239 tlb_info[i].ti_ctx = 0; 1240 tlb_info[i].ti_flags = 0; 1241 tlbnext = i; 1242 /* Successful flushes */ 1243 tlbflush_ev.ev_count++; 1244 } 1245 } 1246 1247 void 1248 ppc4xx_tlb_flush_all(void) 1249 { 1250 u_long i; 1251 1252 for (i = 0; i < NTLB; i++) 1253 if (!TLB_LOCKED(i)) { 1254 __asm volatile( 1255 "tlbwe %0,%1,0;" : : "r" (0), "r" (i)); 1256 tlb_info[i].ti_ctx = 0; 1257 tlb_info[i].ti_flags = 0; 1258 } 1259 1260 __asm volatile("isync"); 1261 } 1262 1263 /* Find a TLB entry to evict. */ 1264 static int 1265 ppc4xx_tlb_find_victim(void) 1266 { 1267 int flags; 1268 1269 for (;;) { 1270 if (++tlbnext >= NTLB) 1271 tlbnext = tlb_nreserved; 1272 flags = tlb_info[tlbnext].ti_flags; 1273 if (!(flags & TLBF_USED) || 1274 (flags & (TLBF_LOCKED | TLBF_REF)) == 0) { 1275 u_long va, stack = (u_long)&va; 1276 1277 if (!((tlb_info[tlbnext].ti_va ^ stack) & (~PGOFSET)) && 1278 (tlb_info[tlbnext].ti_ctx == KERNEL_PID) && 1279 (flags & TLBF_USED)) { 1280 /* Kernel stack page */ 1281 flags |= TLBF_REF; 1282 tlb_info[tlbnext].ti_flags = flags; 1283 } else { 1284 /* Found it! */ 1285 return (tlbnext); 1286 } 1287 } else { 1288 tlb_info[tlbnext].ti_flags = (flags & ~TLBF_REF); 1289 } 1290 } 1291 } 1292 1293 void 1294 ppc4xx_tlb_enter(int ctx, vaddr_t va, u_int pte) 1295 { 1296 u_long th, tl, idx; 1297 int msr, pid; 1298 paddr_t pa; 1299 int sz; 1300 1301 tlbenter_ev.ev_count++; 1302 1303 sz = (pte & TTE_SZ_MASK) >> TTE_SZ_SHIFT; 1304 pa = (pte & TTE_RPN_MASK(sz)); 1305 th = (va & TLB_EPN_MASK) | (sz << TLB_SIZE_SHFT) | TLB_VALID; 1306 tl = (pte & ~TLB_RPN_MASK) | pa; 1307 tl |= ppc4xx_tlbflags(va, pa); 1308 1309 idx = ppc4xx_tlb_find_victim(); 1310 1311 #ifdef DIAGNOSTIC 1312 if ((idx < tlb_nreserved) || (idx >= NTLB) || (idx & 63) == 0) { 1313 panic("ppc4xx_tlb_enter: replacing entry %ld", idx); 1314 } 1315 #endif 1316 1317 tlb_info[idx].ti_va = (va & TLB_EPN_MASK); 1318 tlb_info[idx].ti_ctx = ctx; 1319 tlb_info[idx].ti_flags = TLBF_USED | TLBF_REF; 1320 1321 __asm volatile( 1322 "mfmsr %0;" /* Save MSR */ 1323 "li %1,0;" 1324 "mtmsr %1;" /* Clear MSR */ 1325 "isync;" 1326 "tlbwe %1,%3,0;" /* Invalidate old entry. */ 1327 "mfpid %1;" /* Save old PID */ 1328 "mtpid %2;" /* Load translation ctx */ 1329 "isync;" 1330 "tlbwe %4,%3,1; tlbwe %5,%3,0;" /* Set TLB */ 1331 "isync;" 1332 "mtpid %1; mtmsr %0;" /* Restore PID and MSR */ 1333 "isync;" 1334 : "=&r" (msr), "=&r" (pid) 1335 : "r" (ctx), "r" (idx), "r" (tl), "r" (th)); 1336 } 1337 1338 void 1339 ppc4xx_tlb_init(void) 1340 { 1341 int i; 1342 1343 /* Mark reserved TLB entries */ 1344 for (i = 0; i < tlb_nreserved; i++) { 1345 tlb_info[i].ti_flags = TLBF_LOCKED | TLBF_USED; 1346 tlb_info[i].ti_ctx = KERNEL_PID; 1347 } 1348 1349 /* Setup security zones */ 1350 /* Z0 - accessible by kernel only if TLB entry permissions allow 1351 * Z1,Z2 - access is controlled by TLB entry permissions 1352 * Z3 - full access regardless of TLB entry permissions 1353 */ 1354 1355 __asm volatile( 1356 "mtspr %0,%1;" 1357 "isync;" 1358 :: "K"(SPR_ZPR), "r" (0x1b000000)); 1359 } 1360 1361 /* 1362 * ppc4xx_tlb_size_mask: 1363 * 1364 * Roundup size to supported page size, return TLBHI mask and real size. 1365 */ 1366 static int 1367 ppc4xx_tlb_size_mask(size_t size, int *mask, int *rsiz) 1368 { 1369 int i; 1370 1371 for (i = 0; i < __arraycount(tlbsize); i++) 1372 if (size <= tlbsize[i]) { 1373 *mask = (i << TLB_SIZE_SHFT); 1374 *rsiz = tlbsize[i]; 1375 return (0); 1376 } 1377 return (EINVAL); 1378 } 1379 1380 /* 1381 * ppc4xx_tlb_mapiodev: 1382 * 1383 * Lookup virtual address of mapping previously entered via 1384 * ppc4xx_tlb_reserve. Search TLB directly so that we don't 1385 * need to waste extra storage for reserved mappings. Note 1386 * that reading TLBHI also sets PID, but all reserved mappings 1387 * use KERNEL_PID, so the side effect is nil. 1388 */ 1389 void * 1390 ppc4xx_tlb_mapiodev(paddr_t base, psize_t len) 1391 { 1392 paddr_t pa; 1393 vaddr_t va; 1394 u_int lo, hi, sz; 1395 int i; 1396 1397 /* tlb_nreserved is only allowed to grow, so this is safe. */ 1398 for (i = 0; i < tlb_nreserved; i++) { 1399 __asm volatile ( 1400 " tlbre %0,%2,1 \n" /* TLBLO */ 1401 " tlbre %1,%2,0 \n" /* TLBHI */ 1402 : "=&r" (lo), "=&r" (hi) 1403 : "r" (i)); 1404 1405 KASSERT(hi & TLB_VALID); 1406 KASSERT(mfspr(SPR_PID) == KERNEL_PID); 1407 1408 pa = (lo & TLB_RPN_MASK); 1409 if (base < pa) 1410 continue; 1411 1412 sz = tlbsize[(hi & TLB_SIZE_MASK) >> TLB_SIZE_SHFT]; 1413 if ((base + len) > (pa + sz)) 1414 continue; 1415 1416 va = (hi & TLB_EPN_MASK) + (base & (sz - 1)); /* sz = 2^n */ 1417 return (void *)(va); 1418 } 1419 1420 return (NULL); 1421 } 1422 1423 /* 1424 * ppc4xx_tlb_reserve: 1425 * 1426 * Map physical range to kernel virtual chunk via reserved TLB entry. 1427 */ 1428 void 1429 ppc4xx_tlb_reserve(paddr_t pa, vaddr_t va, size_t size, int flags) 1430 { 1431 u_int lo, hi; 1432 int szmask, rsize; 1433 1434 /* Called before pmap_bootstrap(), va outside kernel space. */ 1435 KASSERT(va < VM_MIN_KERNEL_ADDRESS || va >= VM_MAX_KERNEL_ADDRESS); 1436 KASSERT(! pmap_bootstrap_done); 1437 KASSERT(tlb_nreserved < NTLB); 1438 1439 /* Resolve size. */ 1440 if (ppc4xx_tlb_size_mask(size, &szmask, &rsize) != 0) 1441 panic("ppc4xx_tlb_reserve: entry %d, %zuB too large", 1442 size, tlb_nreserved); 1443 1444 /* Real size will be power of two >= 1024, so this is OK. */ 1445 pa &= ~(rsize - 1); /* RPN */ 1446 va &= ~(rsize - 1); /* EPN */ 1447 1448 lo = pa | TLB_WR | flags; 1449 hi = va | TLB_VALID | szmask; 1450 1451 #ifdef PPC_4XX_NOCACHE 1452 lo |= TLB_I; 1453 #endif 1454 1455 __asm volatile( 1456 " tlbwe %1,%0,1 \n" /* write TLBLO */ 1457 " tlbwe %2,%0,0 \n" /* write TLBHI */ 1458 " isync \n" 1459 : : "r" (tlb_nreserved), "r" (lo), "r" (hi)); 1460 1461 tlb_nreserved++; 1462 } 1463 1464 /* 1465 * We should pass the ctx in from trap code. 1466 */ 1467 int 1468 pmap_tlbmiss(vaddr_t va, int ctx) 1469 { 1470 volatile u_int *pte; 1471 u_long tte; 1472 1473 tlbmiss_ev.ev_count++; 1474 1475 /* 1476 * We will reserve 0 upto VM_MIN_KERNEL_ADDRESS for va == pa mappings. 1477 * Physical RAM is expected to live in this range, care must be taken 1478 * to not clobber 0 upto ${physmem} with device mappings in machdep 1479 * code. 1480 */ 1481 if (ctx != KERNEL_PID || 1482 (va >= VM_MIN_KERNEL_ADDRESS && va < VM_MAX_KERNEL_ADDRESS)) { 1483 pte = pte_find((struct pmap *)__UNVOLATILE(ctxbusy[ctx]), va); 1484 if (pte == NULL) { 1485 /* Map unmanaged addresses directly for kernel access */ 1486 return 1; 1487 } 1488 tte = *pte; 1489 if (tte == 0) { 1490 return 1; 1491 } 1492 } else { 1493 /* Create a 16MB writable mapping. */ 1494 #ifdef PPC_4XX_NOCACHE 1495 tte = TTE_PA(va) | TTE_ZONE(ZONE_PRIV) | TTE_SZ_16M | TTE_I |TTE_WR; 1496 #else 1497 tte = TTE_PA(va) | TTE_ZONE(ZONE_PRIV) | TTE_SZ_16M | TTE_WR; 1498 #endif 1499 } 1500 ppc4xx_tlb_enter(ctx, va, tte); 1501 1502 return 0; 1503 } 1504 1505 /* 1506 * Flush all the entries matching a context from the TLB. 1507 */ 1508 static int 1509 ctx_flush(int cnum) 1510 { 1511 int i; 1512 1513 /* We gotta steal this context */ 1514 for (i = tlb_nreserved; i < NTLB; i++) { 1515 if (tlb_info[i].ti_ctx == cnum) { 1516 /* Can't steal ctx if it has a locked entry. */ 1517 if (TLB_LOCKED(i)) { 1518 #ifdef DIAGNOSTIC 1519 printf("ctx_flush: can't invalidate " 1520 "locked mapping %d " 1521 "for context %d\n", i, cnum); 1522 #ifdef DDB 1523 Debugger(); 1524 #endif 1525 #endif 1526 return (1); 1527 } 1528 #ifdef DIAGNOSTIC 1529 if (i < tlb_nreserved) 1530 panic("TLB entry %d not locked", i); 1531 #endif 1532 /* Invalidate particular TLB entry regardless of locked status */ 1533 __asm volatile("tlbwe %0,%1,0" : :"r"(0),"r"(i)); 1534 tlb_info[i].ti_ctx = 0; 1535 tlb_info[i].ti_flags = 0; 1536 } 1537 } 1538 return (0); 1539 } 1540 1541 /* 1542 * Allocate a context. If necessary, steal one from someone else. 1543 * 1544 * The new context is flushed from the TLB before returning. 1545 */ 1546 int 1547 ctx_alloc(struct pmap *pm) 1548 { 1549 int s, cnum; 1550 static int next = MINCTX; 1551 1552 if (pm == pmap_kernel()) { 1553 #ifdef DIAGNOSTIC 1554 printf("ctx_alloc: kernel pmap!\n"); 1555 #endif 1556 return (0); 1557 } 1558 s = splvm(); 1559 1560 /* Find a likely context. */ 1561 cnum = next; 1562 do { 1563 if ((++cnum) >= NUMCTX) 1564 cnum = MINCTX; 1565 } while (ctxbusy[cnum] != NULL && cnum != next); 1566 1567 /* Now clean it out */ 1568 oops: 1569 if (cnum < MINCTX) 1570 cnum = MINCTX; /* Never steal ctx 0 or 1 */ 1571 if (ctx_flush(cnum)) { 1572 /* oops -- something's wired. */ 1573 if ((++cnum) >= NUMCTX) 1574 cnum = MINCTX; 1575 goto oops; 1576 } 1577 1578 if (ctxbusy[cnum]) { 1579 #ifdef DEBUG 1580 /* We should identify this pmap and clear it */ 1581 printf("Warning: stealing context %d\n", cnum); 1582 #endif 1583 ctxbusy[cnum]->pm_ctx = 0; 1584 } 1585 ctxbusy[cnum] = pm; 1586 next = cnum; 1587 splx(s); 1588 pm->pm_ctx = cnum; 1589 1590 return cnum; 1591 } 1592 1593 /* 1594 * Give away a context. 1595 */ 1596 void 1597 ctx_free(struct pmap *pm) 1598 { 1599 int oldctx; 1600 1601 oldctx = pm->pm_ctx; 1602 1603 if (oldctx == 0) 1604 panic("ctx_free: freeing kernel context"); 1605 #ifdef DIAGNOSTIC 1606 if (ctxbusy[oldctx] == 0) 1607 printf("ctx_free: freeing free context %d\n", oldctx); 1608 if (ctxbusy[oldctx] != pm) { 1609 printf("ctx_free: freeing someone esle's context\n " 1610 "ctxbusy[%d] = %p, pm->pm_ctx = %p\n", 1611 oldctx, (void *)(u_long)ctxbusy[oldctx], pm); 1612 #ifdef DDB 1613 Debugger(); 1614 #endif 1615 } 1616 #endif 1617 /* We should verify it has not been stolen and reallocated... */ 1618 ctxbusy[oldctx] = NULL; 1619 ctx_flush(oldctx); 1620 } 1621 1622 1623 #ifdef DEBUG 1624 /* 1625 * Test ref/modify handling. 1626 */ 1627 void pmap_testout(void); 1628 void 1629 pmap_testout(void) 1630 { 1631 vaddr_t va; 1632 volatile int *loc; 1633 int val = 0; 1634 paddr_t pa; 1635 struct vm_page *pg; 1636 int ref, mod; 1637 1638 /* Allocate a page */ 1639 va = (vaddr_t)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 1640 UVM_KMF_WIRED | UVM_KMF_ZERO); 1641 loc = (int*)va; 1642 1643 pmap_extract(pmap_kernel(), va, &pa); 1644 pg = PHYS_TO_VM_PAGE(pa); 1645 pmap_unwire(pmap_kernel(), va); 1646 1647 pmap_kremove(va, PAGE_SIZE); 1648 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1649 pmap_update(pmap_kernel()); 1650 1651 /* Now clear reference and modify */ 1652 ref = pmap_clear_reference(pg); 1653 mod = pmap_clear_modify(pg); 1654 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1655 (void *)(u_long)va, (long)pa, 1656 ref, mod); 1657 1658 /* Check it's properly cleared */ 1659 ref = pmap_is_referenced(pg); 1660 mod = pmap_is_modified(pg); 1661 printf("Checking cleared page: ref %d, mod %d\n", 1662 ref, mod); 1663 1664 /* Reference page */ 1665 val = *loc; 1666 1667 ref = pmap_is_referenced(pg); 1668 mod = pmap_is_modified(pg); 1669 printf("Referenced page: ref %d, mod %d val %x\n", 1670 ref, mod, val); 1671 1672 /* Now clear reference and modify */ 1673 ref = pmap_clear_reference(pg); 1674 mod = pmap_clear_modify(pg); 1675 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1676 (void *)(u_long)va, (long)pa, 1677 ref, mod); 1678 1679 /* Modify page */ 1680 *loc = 1; 1681 1682 ref = pmap_is_referenced(pg); 1683 mod = pmap_is_modified(pg); 1684 printf("Modified page: ref %d, mod %d\n", 1685 ref, mod); 1686 1687 /* Now clear reference and modify */ 1688 ref = pmap_clear_reference(pg); 1689 mod = pmap_clear_modify(pg); 1690 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1691 (void *)(u_long)va, (long)pa, 1692 ref, mod); 1693 1694 /* Check it's properly cleared */ 1695 ref = pmap_is_referenced(pg); 1696 mod = pmap_is_modified(pg); 1697 printf("Checking cleared page: ref %d, mod %d\n", 1698 ref, mod); 1699 1700 /* Modify page */ 1701 *loc = 1; 1702 1703 ref = pmap_is_referenced(pg); 1704 mod = pmap_is_modified(pg); 1705 printf("Modified page: ref %d, mod %d\n", 1706 ref, mod); 1707 1708 /* Check pmap_protect() */ 1709 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_READ); 1710 pmap_update(pmap_kernel()); 1711 ref = pmap_is_referenced(pg); 1712 mod = pmap_is_modified(pg); 1713 printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n", 1714 ref, mod); 1715 1716 /* Now clear reference and modify */ 1717 ref = pmap_clear_reference(pg); 1718 mod = pmap_clear_modify(pg); 1719 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1720 (void *)(u_long)va, (long)pa, 1721 ref, mod); 1722 1723 /* Reference page */ 1724 val = *loc; 1725 1726 ref = pmap_is_referenced(pg); 1727 mod = pmap_is_modified(pg); 1728 printf("Referenced page: ref %d, mod %d val %x\n", 1729 ref, mod, val); 1730 1731 /* Now clear reference and modify */ 1732 ref = pmap_clear_reference(pg); 1733 mod = pmap_clear_modify(pg); 1734 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1735 (void *)(u_long)va, (long)pa, 1736 ref, mod); 1737 1738 /* Modify page */ 1739 #if 0 1740 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1741 pmap_update(pmap_kernel()); 1742 #endif 1743 *loc = 1; 1744 1745 ref = pmap_is_referenced(pg); 1746 mod = pmap_is_modified(pg); 1747 printf("Modified page: ref %d, mod %d\n", 1748 ref, mod); 1749 1750 /* Check pmap_protect() */ 1751 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_NONE); 1752 pmap_update(pmap_kernel()); 1753 ref = pmap_is_referenced(pg); 1754 mod = pmap_is_modified(pg); 1755 printf("pmap_protect(): ref %d, mod %d\n", 1756 ref, mod); 1757 1758 /* Now clear reference and modify */ 1759 ref = pmap_clear_reference(pg); 1760 mod = pmap_clear_modify(pg); 1761 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1762 (void *)(u_long)va, (long)pa, 1763 ref, mod); 1764 1765 /* Reference page */ 1766 val = *loc; 1767 1768 ref = pmap_is_referenced(pg); 1769 mod = pmap_is_modified(pg); 1770 printf("Referenced page: ref %d, mod %d val %x\n", 1771 ref, mod, val); 1772 1773 /* Now clear reference and modify */ 1774 ref = pmap_clear_reference(pg); 1775 mod = pmap_clear_modify(pg); 1776 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1777 (void *)(u_long)va, (long)pa, 1778 ref, mod); 1779 1780 /* Modify page */ 1781 #if 0 1782 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1783 pmap_update(pmap_kernel()); 1784 #endif 1785 *loc = 1; 1786 1787 ref = pmap_is_referenced(pg); 1788 mod = pmap_is_modified(pg); 1789 printf("Modified page: ref %d, mod %d\n", 1790 ref, mod); 1791 1792 /* Check pmap_pag_protect() */ 1793 pmap_page_protect(pg, VM_PROT_READ); 1794 ref = pmap_is_referenced(pg); 1795 mod = pmap_is_modified(pg); 1796 printf("pmap_page_protect(VM_PROT_READ): ref %d, mod %d\n", 1797 ref, mod); 1798 1799 /* Now clear reference and modify */ 1800 ref = pmap_clear_reference(pg); 1801 mod = pmap_clear_modify(pg); 1802 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1803 (void *)(u_long)va, (long)pa, 1804 ref, mod); 1805 1806 /* Reference page */ 1807 val = *loc; 1808 1809 ref = pmap_is_referenced(pg); 1810 mod = pmap_is_modified(pg); 1811 printf("Referenced page: ref %d, mod %d val %x\n", 1812 ref, mod, val); 1813 1814 /* Now clear reference and modify */ 1815 ref = pmap_clear_reference(pg); 1816 mod = pmap_clear_modify(pg); 1817 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1818 (void *)(u_long)va, (long)pa, 1819 ref, mod); 1820 1821 /* Modify page */ 1822 #if 0 1823 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1824 pmap_update(pmap_kernel()); 1825 #endif 1826 *loc = 1; 1827 1828 ref = pmap_is_referenced(pg); 1829 mod = pmap_is_modified(pg); 1830 printf("Modified page: ref %d, mod %d\n", 1831 ref, mod); 1832 1833 /* Check pmap_pag_protect() */ 1834 pmap_page_protect(pg, VM_PROT_NONE); 1835 ref = pmap_is_referenced(pg); 1836 mod = pmap_is_modified(pg); 1837 printf("pmap_page_protect(): ref %d, mod %d\n", 1838 ref, mod); 1839 1840 /* Now clear reference and modify */ 1841 ref = pmap_clear_reference(pg); 1842 mod = pmap_clear_modify(pg); 1843 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1844 (void *)(u_long)va, (long)pa, 1845 ref, mod); 1846 1847 1848 /* Reference page */ 1849 val = *loc; 1850 1851 ref = pmap_is_referenced(pg); 1852 mod = pmap_is_modified(pg); 1853 printf("Referenced page: ref %d, mod %d val %x\n", 1854 ref, mod, val); 1855 1856 /* Now clear reference and modify */ 1857 ref = pmap_clear_reference(pg); 1858 mod = pmap_clear_modify(pg); 1859 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1860 (void *)(u_long)va, (long)pa, 1861 ref, mod); 1862 1863 /* Modify page */ 1864 #if 0 1865 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1866 pmap_update(pmap_kernel()); 1867 #endif 1868 *loc = 1; 1869 1870 ref = pmap_is_referenced(pg); 1871 mod = pmap_is_modified(pg); 1872 printf("Modified page: ref %d, mod %d\n", 1873 ref, mod); 1874 1875 /* Unmap page */ 1876 pmap_remove(pmap_kernel(), va, va+1); 1877 pmap_update(pmap_kernel()); 1878 ref = pmap_is_referenced(pg); 1879 mod = pmap_is_modified(pg); 1880 printf("Unmapped page: ref %d, mod %d\n", ref, mod); 1881 1882 /* Now clear reference and modify */ 1883 ref = pmap_clear_reference(pg); 1884 mod = pmap_clear_modify(pg); 1885 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1886 (void *)(u_long)va, (long)pa, ref, mod); 1887 1888 /* Check it's properly cleared */ 1889 ref = pmap_is_referenced(pg); 1890 mod = pmap_is_modified(pg); 1891 printf("Checking cleared page: ref %d, mod %d\n", 1892 ref, mod); 1893 1894 pmap_remove(pmap_kernel(), va, va + PAGE_SIZE); 1895 pmap_kenter_pa(va, pa, VM_PROT_ALL, 0); 1896 uvm_km_free(kernel_map, (vaddr_t)va, PAGE_SIZE, UVM_KMF_WIRED); 1897 } 1898 #endif 1899