1 /* $NetBSD: pmap.c,v 1.72 2012/01/27 19:48:39 para Exp $ */ 2 3 /* 4 * Copyright 2001 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Eduardo Horvath and Simon Burge for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 40 * Copyright (C) 1995, 1996 TooLs GmbH. 41 * All rights reserved. 42 * 43 * Redistribution and use in source and binary forms, with or without 44 * modification, are permitted provided that the following conditions 45 * are met: 46 * 1. Redistributions of source code must retain the above copyright 47 * notice, this list of conditions and the following disclaimer. 48 * 2. Redistributions in binary form must reproduce the above copyright 49 * notice, this list of conditions and the following disclaimer in the 50 * documentation and/or other materials provided with the distribution. 51 * 3. All advertising materials mentioning features or use of this software 52 * must display the following acknowledgement: 53 * This product includes software developed by TooLs GmbH. 54 * 4. The name of TooLs GmbH may not be used to endorse or promote products 55 * derived from this software without specific prior written permission. 56 * 57 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 58 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 59 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 60 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 61 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 62 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 63 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 64 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 65 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 66 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 67 */ 68 69 #include <sys/cdefs.h> 70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.72 2012/01/27 19:48:39 para Exp $"); 71 72 #include <sys/param.h> 73 #include <sys/cpu.h> 74 #include <sys/device.h> 75 #include <sys/kmem.h> 76 #include <sys/pool.h> 77 #include <sys/proc.h> 78 #include <sys/queue.h> 79 #include <sys/systm.h> 80 81 #include <uvm/uvm.h> 82 83 #include <machine/powerpc.h> 84 #include <machine/tlb.h> 85 86 #include <powerpc/pcb.h> 87 88 #include <powerpc/spr.h> 89 #include <powerpc/ibm4xx/spr.h> 90 91 #include <powerpc/ibm4xx/cpu.h> 92 93 /* 94 * kernmap is an array of PTEs large enough to map in 95 * 4GB. At 16KB/page it is 256K entries or 2MB. 96 */ 97 #define KERNMAP_SIZE ((0xffffffffU/PAGE_SIZE)+1) 98 void *kernmap; 99 100 #define MINCTX 2 101 #define NUMCTX 256 102 103 volatile struct pmap *ctxbusy[NUMCTX]; 104 105 #define TLBF_USED 0x1 106 #define TLBF_REF 0x2 107 #define TLBF_LOCKED 0x4 108 #define TLB_LOCKED(i) (tlb_info[(i)].ti_flags & TLBF_LOCKED) 109 110 typedef struct tlb_info_s { 111 char ti_flags; 112 char ti_ctx; /* TLB_PID assiciated with the entry */ 113 u_int ti_va; 114 } tlb_info_t; 115 116 volatile tlb_info_t tlb_info[NTLB]; 117 /* We'll use a modified FIFO replacement policy cause it's cheap */ 118 volatile int tlbnext; 119 120 static int tlb_nreserved = 0; 121 static int pmap_bootstrap_done = 0; 122 123 /* Event counters */ 124 struct evcnt tlbmiss_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 125 NULL, "cpu", "tlbmiss"); 126 struct evcnt tlbhit_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 127 NULL, "cpu", "tlbhit"); 128 struct evcnt tlbflush_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 129 NULL, "cpu", "tlbflush"); 130 struct evcnt tlbenter_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 131 NULL, "cpu", "tlbenter"); 132 EVCNT_ATTACH_STATIC(tlbmiss_ev); 133 EVCNT_ATTACH_STATIC(tlbhit_ev); 134 EVCNT_ATTACH_STATIC(tlbflush_ev); 135 EVCNT_ATTACH_STATIC(tlbenter_ev); 136 137 struct pmap kernel_pmap_; 138 struct pmap *const kernel_pmap_ptr = &kernel_pmap_; 139 140 static int npgs; 141 static u_int nextavail; 142 #ifndef MSGBUFADDR 143 extern paddr_t msgbuf_paddr; 144 #endif 145 146 static struct mem_region *mem, *avail; 147 148 /* 149 * This is a cache of referenced/modified bits. 150 * Bits herein are shifted by ATTRSHFT. 151 */ 152 static char *pmap_attrib; 153 154 #define PV_WIRED 0x1 155 #define PV_WIRE(pv) ((pv)->pv_va |= PV_WIRED) 156 #define PV_UNWIRE(pv) ((pv)->pv_va &= ~PV_WIRED) 157 #define PV_ISWIRED(pv) ((pv)->pv_va & PV_WIRED) 158 #define PV_CMPVA(va,pv) (!(((pv)->pv_va ^ (va)) & (~PV_WIRED))) 159 160 struct pv_entry { 161 struct pv_entry *pv_next; /* Linked list of mappings */ 162 struct pmap *pv_pm; 163 vaddr_t pv_va; /* virtual address of mapping */ 164 }; 165 166 /* Each index corresponds to TLB_SIZE_* value. */ 167 static size_t tlbsize[] = { 168 1024, /* TLB_SIZE_1K */ 169 4096, /* TLB_SIZE_4K */ 170 16384, /* TLB_SIZE_16K */ 171 65536, /* TLB_SIZE_64K */ 172 262144, /* TLB_SIZE_256K */ 173 1048576, /* TLB_SIZE_1M */ 174 4194304, /* TLB_SIZE_4M */ 175 16777216, /* TLB_SIZE_16M */ 176 }; 177 178 struct pv_entry *pv_table; 179 static struct pool pv_pool; 180 181 static int pmap_initialized; 182 183 static int ctx_flush(int); 184 185 struct pv_entry *pa_to_pv(paddr_t); 186 static inline char *pa_to_attr(paddr_t); 187 188 static inline volatile u_int *pte_find(struct pmap *, vaddr_t); 189 static inline int pte_enter(struct pmap *, vaddr_t, u_int); 190 191 static inline int pmap_enter_pv(struct pmap *, vaddr_t, paddr_t, int); 192 static void pmap_remove_pv(struct pmap *, vaddr_t, paddr_t); 193 194 static int ppc4xx_tlb_size_mask(size_t, int *, int *); 195 196 197 struct pv_entry * 198 pa_to_pv(paddr_t pa) 199 { 200 int bank, pg; 201 202 bank = vm_physseg_find(atop(pa), &pg); 203 if (bank == -1) 204 return NULL; 205 return &VM_PHYSMEM_PTR(bank)->pmseg.pvent[pg]; 206 } 207 208 static inline char * 209 pa_to_attr(paddr_t pa) 210 { 211 int bank, pg; 212 213 bank = vm_physseg_find(atop(pa), &pg); 214 if (bank == -1) 215 return NULL; 216 return &VM_PHYSMEM_PTR(bank)->pmseg.attrs[pg]; 217 } 218 219 /* 220 * Insert PTE into page table. 221 */ 222 int 223 pte_enter(struct pmap *pm, vaddr_t va, u_int pte) 224 { 225 int seg = STIDX(va); 226 int ptn = PTIDX(va); 227 u_int oldpte; 228 229 if (!pm->pm_ptbl[seg]) { 230 /* Don't allocate a page to clear a non-existent mapping. */ 231 if (!pte) 232 return (0); 233 /* Allocate a page XXXX this will sleep! */ 234 pm->pm_ptbl[seg] = 235 (uint *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 236 UVM_KMF_WIRED | UVM_KMF_ZERO); 237 } 238 oldpte = pm->pm_ptbl[seg][ptn]; 239 pm->pm_ptbl[seg][ptn] = pte; 240 241 /* Flush entry. */ 242 ppc4xx_tlb_flush(va, pm->pm_ctx); 243 if (oldpte != pte) { 244 if (pte == 0) 245 pm->pm_stats.resident_count--; 246 else 247 pm->pm_stats.resident_count++; 248 } 249 return (1); 250 } 251 252 /* 253 * Get a pointer to a PTE in a page table. 254 */ 255 volatile u_int * 256 pte_find(struct pmap *pm, vaddr_t va) 257 { 258 int seg = STIDX(va); 259 int ptn = PTIDX(va); 260 261 if (pm->pm_ptbl[seg]) 262 return (&pm->pm_ptbl[seg][ptn]); 263 264 return (NULL); 265 } 266 267 /* 268 * This is called during initppc, before the system is really initialized. 269 */ 270 void 271 pmap_bootstrap(u_int kernelstart, u_int kernelend) 272 { 273 struct mem_region *mp, *mp1; 274 int cnt, i; 275 u_int s, e, sz; 276 277 tlbnext = tlb_nreserved; 278 279 /* 280 * Allocate the kernel page table at the end of 281 * kernel space so it's in the locked TTE. 282 */ 283 kernmap = (void *)kernelend; 284 285 /* 286 * Initialize kernel page table. 287 */ 288 for (i = 0; i < STSZ; i++) { 289 pmap_kernel()->pm_ptbl[i] = 0; 290 } 291 ctxbusy[0] = ctxbusy[1] = pmap_kernel(); 292 293 /* 294 * Announce page-size to the VM-system 295 */ 296 uvmexp.pagesize = NBPG; 297 uvm_setpagesize(); 298 299 /* 300 * Get memory. 301 */ 302 mem_regions(&mem, &avail); 303 for (mp = mem; mp->size; mp++) { 304 physmem += btoc(mp->size); 305 printf("+%lx,",mp->size); 306 } 307 printf("\n"); 308 ppc4xx_tlb_init(); 309 /* 310 * Count the number of available entries. 311 */ 312 for (cnt = 0, mp = avail; mp->size; mp++) 313 cnt++; 314 315 /* 316 * Page align all regions. 317 * Non-page aligned memory isn't very interesting to us. 318 * Also, sort the entries for ascending addresses. 319 */ 320 kernelstart &= ~PGOFSET; 321 kernelend = (kernelend + PGOFSET) & ~PGOFSET; 322 for (mp = avail; mp->size; mp++) { 323 s = mp->start; 324 e = mp->start + mp->size; 325 printf("%08x-%08x -> ",s,e); 326 /* 327 * Check whether this region holds all of the kernel. 328 */ 329 if (s < kernelstart && e > kernelend) { 330 avail[cnt].start = kernelend; 331 avail[cnt++].size = e - kernelend; 332 e = kernelstart; 333 } 334 /* 335 * Look whether this regions starts within the kernel. 336 */ 337 if (s >= kernelstart && s < kernelend) { 338 if (e <= kernelend) 339 goto empty; 340 s = kernelend; 341 } 342 /* 343 * Now look whether this region ends within the kernel. 344 */ 345 if (e > kernelstart && e <= kernelend) { 346 if (s >= kernelstart) 347 goto empty; 348 e = kernelstart; 349 } 350 /* 351 * Now page align the start and size of the region. 352 */ 353 s = round_page(s); 354 e = trunc_page(e); 355 if (e < s) 356 e = s; 357 sz = e - s; 358 printf("%08x-%08x = %x\n",s,e,sz); 359 /* 360 * Check whether some memory is left here. 361 */ 362 if (sz == 0) { 363 empty: 364 memmove(mp, mp + 1, 365 (cnt - (mp - avail)) * sizeof *mp); 366 cnt--; 367 mp--; 368 continue; 369 } 370 /* 371 * Do an insertion sort. 372 */ 373 npgs += btoc(sz); 374 for (mp1 = avail; mp1 < mp; mp1++) 375 if (s < mp1->start) 376 break; 377 if (mp1 < mp) { 378 memmove(mp1 + 1, mp1, (char *)mp - (char *)mp1); 379 mp1->start = s; 380 mp1->size = sz; 381 } else { 382 mp->start = s; 383 mp->size = sz; 384 } 385 } 386 387 /* 388 * We cannot do pmap_steal_memory here, 389 * since we don't run with translation enabled yet. 390 */ 391 #ifndef MSGBUFADDR 392 /* 393 * allow for msgbuf 394 */ 395 sz = round_page(MSGBUFSIZE); 396 mp = NULL; 397 for (mp1 = avail; mp1->size; mp1++) 398 if (mp1->size >= sz) 399 mp = mp1; 400 if (mp == NULL) 401 panic("not enough memory?"); 402 403 npgs -= btoc(sz); 404 msgbuf_paddr = mp->start + mp->size - sz; 405 mp->size -= sz; 406 if (mp->size <= 0) 407 memmove(mp, mp + 1, (cnt - (mp - avail)) * sizeof *mp); 408 #endif 409 410 for (mp = avail; mp->size; mp++) 411 uvm_page_physload(atop(mp->start), atop(mp->start + mp->size), 412 atop(mp->start), atop(mp->start + mp->size), 413 VM_FREELIST_DEFAULT); 414 415 /* 416 * Initialize kernel pmap and hardware. 417 */ 418 /* Setup TLB pid allocator so it knows we alreadu using PID 1 */ 419 pmap_kernel()->pm_ctx = KERNEL_PID; 420 nextavail = avail->start; 421 422 pmap_bootstrap_done = 1; 423 } 424 425 /* 426 * Restrict given range to physical memory 427 * 428 * (Used by /dev/mem) 429 */ 430 void 431 pmap_real_memory(paddr_t *start, psize_t *size) 432 { 433 struct mem_region *mp; 434 435 for (mp = mem; mp->size; mp++) { 436 if (*start + *size > mp->start && 437 *start < mp->start + mp->size) { 438 if (*start < mp->start) { 439 *size -= mp->start - *start; 440 *start = mp->start; 441 } 442 if (*start + *size > mp->start + mp->size) 443 *size = mp->start + mp->size - *start; 444 return; 445 } 446 } 447 *size = 0; 448 } 449 450 /* 451 * Initialize anything else for pmap handling. 452 * Called during vm_init(). 453 */ 454 void 455 pmap_init(void) 456 { 457 struct pv_entry *pv; 458 vsize_t sz; 459 vaddr_t addr; 460 int i, s; 461 int bank; 462 char *attr; 463 464 sz = (vsize_t)((sizeof(struct pv_entry) + 1) * npgs); 465 sz = round_page(sz); 466 addr = uvm_km_alloc(kernel_map, sz, 0, UVM_KMF_WIRED | UVM_KMF_ZERO); 467 s = splvm(); 468 pv = pv_table = (struct pv_entry *)addr; 469 for (i = npgs; --i >= 0;) 470 pv++->pv_pm = NULL; 471 pmap_attrib = (char *)pv; 472 memset(pv, 0, npgs); 473 474 pv = pv_table; 475 attr = pmap_attrib; 476 for (bank = 0; bank < vm_nphysseg; bank++) { 477 sz = VM_PHYSMEM_PTR(bank)->end - VM_PHYSMEM_PTR(bank)->start; 478 VM_PHYSMEM_PTR(bank)->pmseg.pvent = pv; 479 VM_PHYSMEM_PTR(bank)->pmseg.attrs = attr; 480 pv += sz; 481 attr += sz; 482 } 483 484 pmap_initialized = 1; 485 splx(s); 486 487 /* Setup a pool for additional pvlist structures */ 488 pool_init(&pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pv_entry", NULL, 489 IPL_VM); 490 } 491 492 /* 493 * How much virtual space is available to the kernel? 494 */ 495 void 496 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 497 { 498 499 #if 0 500 /* 501 * Reserve one segment for kernel virtual memory 502 */ 503 *start = (vaddr_t)(KERNEL_SR << ADDR_SR_SHFT); 504 *end = *start + SEGMENT_LENGTH; 505 #else 506 *start = (vaddr_t) VM_MIN_KERNEL_ADDRESS; 507 *end = (vaddr_t) VM_MAX_KERNEL_ADDRESS; 508 #endif 509 } 510 511 #ifdef PMAP_GROWKERNEL 512 /* 513 * Preallocate kernel page tables to a specified VA. 514 * This simply loops through the first TTE for each 515 * page table from the beginning of the kernel pmap, 516 * reads the entry, and if the result is 517 * zero (either invalid entry or no page table) it stores 518 * a zero there, populating page tables in the process. 519 * This is not the most efficient technique but i don't 520 * expect it to be called that often. 521 */ 522 extern struct vm_page *vm_page_alloc1(void); 523 extern void vm_page_free1(struct vm_page *); 524 525 vaddr_t kbreak = VM_MIN_KERNEL_ADDRESS; 526 527 vaddr_t 528 pmap_growkernel(vaddr_t maxkvaddr) 529 { 530 int s; 531 int seg; 532 paddr_t pg; 533 struct pmap *pm = pmap_kernel(); 534 535 s = splvm(); 536 537 /* Align with the start of a page table */ 538 for (kbreak &= ~(PTMAP-1); kbreak < maxkvaddr; 539 kbreak += PTMAP) { 540 seg = STIDX(kbreak); 541 542 if (pte_find(pm, kbreak)) 543 continue; 544 545 if (uvm.page_init_done) { 546 pg = (paddr_t)VM_PAGE_TO_PHYS(vm_page_alloc1()); 547 } else { 548 if (!uvm_page_physget(&pg)) 549 panic("pmap_growkernel: no memory"); 550 } 551 if (!pg) 552 panic("pmap_growkernel: no pages"); 553 pmap_zero_page((paddr_t)pg); 554 555 /* XXX This is based on all phymem being addressable */ 556 pm->pm_ptbl[seg] = (u_int *)pg; 557 } 558 splx(s); 559 return (kbreak); 560 } 561 562 /* 563 * vm_page_alloc1: 564 * 565 * Allocate and return a memory cell with no associated object. 566 */ 567 struct vm_page * 568 vm_page_alloc1(void) 569 { 570 struct vm_page *pg; 571 572 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE); 573 if (pg) { 574 pg->wire_count = 1; /* no mappings yet */ 575 pg->flags &= ~PG_BUSY; /* never busy */ 576 } 577 return pg; 578 } 579 580 /* 581 * vm_page_free1: 582 * 583 * Returns the given page to the free list, 584 * disassociating it with any VM object. 585 * 586 * Object and page must be locked prior to entry. 587 */ 588 void 589 vm_page_free1(struct vm_page *pg) 590 { 591 #ifdef DIAGNOSTIC 592 if (pg->flags != (PG_CLEAN|PG_FAKE)) { 593 printf("Freeing invalid page %p\n", pg); 594 printf("pa = %llx\n", (unsigned long long)VM_PAGE_TO_PHYS(pg)); 595 #ifdef DDB 596 Debugger(); 597 #endif 598 return; 599 } 600 #endif 601 pg->flags |= PG_BUSY; 602 pg->wire_count = 0; 603 uvm_pagefree(pg); 604 } 605 #endif 606 607 /* 608 * Create and return a physical map. 609 */ 610 struct pmap * 611 pmap_create(void) 612 { 613 struct pmap *pm; 614 615 pm = kmem_alloc(sizeof(*pm), KM_SLEEP); 616 memset(pm, 0, sizeof *pm); 617 pm->pm_refs = 1; 618 return pm; 619 } 620 621 /* 622 * Add a reference to the given pmap. 623 */ 624 void 625 pmap_reference(struct pmap *pm) 626 { 627 628 pm->pm_refs++; 629 } 630 631 /* 632 * Retire the given pmap from service. 633 * Should only be called if the map contains no valid mappings. 634 */ 635 void 636 pmap_destroy(struct pmap *pm) 637 { 638 int i; 639 640 if (--pm->pm_refs > 0) { 641 return; 642 } 643 KASSERT(pm->pm_stats.resident_count == 0); 644 KASSERT(pm->pm_stats.wired_count == 0); 645 for (i = 0; i < STSZ; i++) 646 if (pm->pm_ptbl[i]) { 647 uvm_km_free(kernel_map, (vaddr_t)pm->pm_ptbl[i], 648 PAGE_SIZE, UVM_KMF_WIRED); 649 pm->pm_ptbl[i] = NULL; 650 } 651 if (pm->pm_ctx) 652 ctx_free(pm); 653 kmem_free(pm, sizeof(*pm)); 654 } 655 656 /* 657 * Copy the range specified by src_addr/len 658 * from the source map to the range dst_addr/len 659 * in the destination map. 660 * 661 * This routine is only advisory and need not do anything. 662 */ 663 void 664 pmap_copy(struct pmap *dst_pmap, struct pmap *src_pmap, vaddr_t dst_addr, 665 vsize_t len, vaddr_t src_addr) 666 { 667 } 668 669 /* 670 * Require that all active physical maps contain no 671 * incorrect entries NOW. 672 */ 673 void 674 pmap_update(struct pmap *pmap) 675 { 676 } 677 678 /* 679 * Fill the given physical page with zeroes. 680 */ 681 void 682 pmap_zero_page(paddr_t pa) 683 { 684 685 #ifdef PPC_4XX_NOCACHE 686 memset((void *)pa, 0, PAGE_SIZE); 687 #else 688 int i; 689 690 for (i = PAGE_SIZE/CACHELINESIZE; i > 0; i--) { 691 __asm volatile ("dcbz 0,%0" :: "r"(pa)); 692 pa += CACHELINESIZE; 693 } 694 #endif 695 } 696 697 /* 698 * Copy the given physical source page to its destination. 699 */ 700 void 701 pmap_copy_page(paddr_t src, paddr_t dst) 702 { 703 704 memcpy((void *)dst, (void *)src, PAGE_SIZE); 705 dcache_wbinv_page(dst); 706 } 707 708 /* 709 * This returns != 0 on success. 710 */ 711 static inline int 712 pmap_enter_pv(struct pmap *pm, vaddr_t va, paddr_t pa, int flags) 713 { 714 struct pv_entry *pv, *npv = NULL; 715 int s; 716 717 if (!pmap_initialized) 718 return 0; 719 720 s = splvm(); 721 pv = pa_to_pv(pa); 722 if (!pv->pv_pm) { 723 /* 724 * No entries yet, use header as the first entry. 725 */ 726 pv->pv_va = va; 727 pv->pv_pm = pm; 728 pv->pv_next = NULL; 729 } else { 730 /* 731 * There is at least one other VA mapping this page. 732 * Place this entry after the header. 733 */ 734 npv = pool_get(&pv_pool, PR_NOWAIT); 735 if (npv == NULL) { 736 if ((flags & PMAP_CANFAIL) == 0) 737 panic("pmap_enter_pv: failed"); 738 splx(s); 739 return 0; 740 } 741 npv->pv_va = va; 742 npv->pv_pm = pm; 743 npv->pv_next = pv->pv_next; 744 pv->pv_next = npv; 745 pv = npv; 746 } 747 if (flags & PMAP_WIRED) { 748 PV_WIRE(pv); 749 pm->pm_stats.wired_count++; 750 } 751 splx(s); 752 return (1); 753 } 754 755 static void 756 pmap_remove_pv(struct pmap *pm, vaddr_t va, paddr_t pa) 757 { 758 struct pv_entry *pv, *npv; 759 760 /* 761 * Remove from the PV table. 762 */ 763 pv = pa_to_pv(pa); 764 if (!pv) 765 return; 766 767 /* 768 * If it is the first entry on the list, it is actually 769 * in the header and we must copy the following entry up 770 * to the header. Otherwise we must search the list for 771 * the entry. In either case we free the now unused entry. 772 */ 773 if (pm == pv->pv_pm && PV_CMPVA(va, pv)) { 774 if (PV_ISWIRED(pv)) { 775 pm->pm_stats.wired_count--; 776 } 777 if ((npv = pv->pv_next)) { 778 *pv = *npv; 779 pool_put(&pv_pool, npv); 780 } else 781 pv->pv_pm = NULL; 782 } else { 783 for (; (npv = pv->pv_next) != NULL; pv = npv) 784 if (pm == npv->pv_pm && PV_CMPVA(va, npv)) 785 break; 786 if (npv) { 787 pv->pv_next = npv->pv_next; 788 if (PV_ISWIRED(npv)) { 789 pm->pm_stats.wired_count--; 790 } 791 pool_put(&pv_pool, npv); 792 } 793 } 794 } 795 796 /* 797 * Insert physical page at pa into the given pmap at virtual address va. 798 */ 799 int 800 pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 801 { 802 int s; 803 u_int tte; 804 bool managed; 805 806 /* 807 * Have to remove any existing mapping first. 808 */ 809 pmap_remove(pm, va, va + PAGE_SIZE); 810 811 if (flags & PMAP_WIRED) 812 flags |= prot; 813 814 managed = uvm_pageismanaged(pa); 815 816 /* 817 * Generate TTE. 818 */ 819 tte = TTE_PA(pa); 820 /* XXXX -- need to support multiple page sizes. */ 821 tte |= TTE_SZ_16K; 822 #ifdef DIAGNOSTIC 823 if ((flags & (PMAP_NOCACHE | PME_WRITETHROUG)) == 824 (PMAP_NOCACHE | PME_WRITETHROUG)) 825 panic("pmap_enter: uncached & writethrough"); 826 #endif 827 if (flags & PMAP_NOCACHE) 828 /* Must be I/O mapping */ 829 tte |= TTE_I | TTE_G; 830 #ifdef PPC_4XX_NOCACHE 831 tte |= TTE_I; 832 #else 833 else if (flags & PME_WRITETHROUG) 834 /* Uncached and writethrough are not compatible */ 835 tte |= TTE_W; 836 #endif 837 if (pm == pmap_kernel()) 838 tte |= TTE_ZONE(ZONE_PRIV); 839 else 840 tte |= TTE_ZONE(ZONE_USER); 841 842 if (flags & VM_PROT_WRITE) 843 tte |= TTE_WR; 844 845 if (flags & VM_PROT_EXECUTE) 846 tte |= TTE_EX; 847 848 /* 849 * Now record mapping for later back-translation. 850 */ 851 if (pmap_initialized && managed) { 852 char *attr; 853 854 if (!pmap_enter_pv(pm, va, pa, flags)) { 855 /* Could not enter pv on a managed page */ 856 return 1; 857 } 858 859 /* Now set attributes. */ 860 attr = pa_to_attr(pa); 861 #ifdef DIAGNOSTIC 862 if (!attr) 863 panic("managed but no attr"); 864 #endif 865 if (flags & VM_PROT_ALL) 866 *attr |= PMAP_ATTR_REF; 867 if (flags & VM_PROT_WRITE) 868 *attr |= PMAP_ATTR_CHG; 869 } 870 871 s = splvm(); 872 873 /* Insert page into page table. */ 874 pte_enter(pm, va, tte); 875 876 /* If this is a real fault, enter it in the tlb */ 877 if (tte && ((flags & PMAP_WIRED) == 0)) { 878 int s2 = splhigh(); 879 ppc4xx_tlb_enter(pm->pm_ctx, va, tte); 880 splx(s2); 881 } 882 splx(s); 883 884 /* Flush the real memory from the instruction cache. */ 885 if ((prot & VM_PROT_EXECUTE) && (tte & TTE_I) == 0) 886 __syncicache((void *)pa, PAGE_SIZE); 887 888 return 0; 889 } 890 891 void 892 pmap_unwire(struct pmap *pm, vaddr_t va) 893 { 894 struct pv_entry *pv; 895 paddr_t pa; 896 int s; 897 898 if (!pmap_extract(pm, va, &pa)) { 899 return; 900 } 901 902 pv = pa_to_pv(pa); 903 if (!pv) 904 return; 905 906 s = splvm(); 907 while (pv != NULL) { 908 if (pm == pv->pv_pm && PV_CMPVA(va, pv)) { 909 if (PV_ISWIRED(pv)) { 910 PV_UNWIRE(pv); 911 pm->pm_stats.wired_count--; 912 } 913 break; 914 } 915 pv = pv->pv_next; 916 } 917 splx(s); 918 } 919 920 void 921 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 922 { 923 int s; 924 u_int tte; 925 struct pmap *pm = pmap_kernel(); 926 927 /* 928 * Have to remove any existing mapping first. 929 */ 930 931 /* 932 * Generate TTE. 933 * 934 * XXXX 935 * 936 * Since the kernel does not handle execution privileges properly, 937 * we will handle read and execute permissions together. 938 */ 939 tte = 0; 940 if (prot & VM_PROT_ALL) { 941 942 tte = TTE_PA(pa) | TTE_EX | TTE_ZONE(ZONE_PRIV); 943 /* XXXX -- need to support multiple page sizes. */ 944 tte |= TTE_SZ_16K; 945 #ifdef DIAGNOSTIC 946 if ((flags & (PMAP_NOCACHE | PME_WRITETHROUG)) == 947 (PMAP_NOCACHE | PME_WRITETHROUG)) 948 panic("pmap_kenter_pa: uncached & writethrough"); 949 #endif 950 if (flags & PMAP_NOCACHE) 951 /* Must be I/O mapping */ 952 tte |= TTE_I | TTE_G; 953 #ifdef PPC_4XX_NOCACHE 954 tte |= TTE_I; 955 #else 956 else if (prot & PME_WRITETHROUG) 957 /* Uncached and writethrough are not compatible */ 958 tte |= TTE_W; 959 #endif 960 if (prot & VM_PROT_WRITE) 961 tte |= TTE_WR; 962 } 963 964 s = splvm(); 965 966 /* Insert page into page table. */ 967 pte_enter(pm, va, tte); 968 splx(s); 969 } 970 971 void 972 pmap_kremove(vaddr_t va, vsize_t len) 973 { 974 975 while (len > 0) { 976 pte_enter(pmap_kernel(), va, 0); 977 va += PAGE_SIZE; 978 len -= PAGE_SIZE; 979 } 980 } 981 982 /* 983 * Remove the given range of mapping entries. 984 */ 985 void 986 pmap_remove(struct pmap *pm, vaddr_t va, vaddr_t endva) 987 { 988 int s; 989 paddr_t pa; 990 volatile u_int *ptp; 991 992 s = splvm(); 993 while (va < endva) { 994 995 if ((ptp = pte_find(pm, va)) && (pa = *ptp)) { 996 pa = TTE_PA(pa); 997 pmap_remove_pv(pm, va, pa); 998 *ptp = 0; 999 ppc4xx_tlb_flush(va, pm->pm_ctx); 1000 pm->pm_stats.resident_count--; 1001 } 1002 va += PAGE_SIZE; 1003 } 1004 1005 splx(s); 1006 } 1007 1008 /* 1009 * Get the physical page address for the given pmap/virtual address. 1010 */ 1011 bool 1012 pmap_extract(struct pmap *pm, vaddr_t va, paddr_t *pap) 1013 { 1014 int seg = STIDX(va); 1015 int ptn = PTIDX(va); 1016 u_int pa = 0; 1017 int s; 1018 1019 s = splvm(); 1020 if (pm->pm_ptbl[seg] && (pa = pm->pm_ptbl[seg][ptn])) { 1021 *pap = TTE_PA(pa) | (va & PGOFSET); 1022 } 1023 splx(s); 1024 return (pa != 0); 1025 } 1026 1027 /* 1028 * Lower the protection on the specified range of this pmap. 1029 * 1030 * There are only two cases: either the protection is going to 0, 1031 * or it is going to read-only. 1032 */ 1033 void 1034 pmap_protect(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1035 { 1036 volatile u_int *ptp; 1037 int s, bic; 1038 1039 if ((prot & VM_PROT_READ) == 0) { 1040 pmap_remove(pm, sva, eva); 1041 return; 1042 } 1043 bic = 0; 1044 if ((prot & VM_PROT_WRITE) == 0) { 1045 bic |= TTE_WR; 1046 } 1047 if ((prot & VM_PROT_EXECUTE) == 0) { 1048 bic |= TTE_EX; 1049 } 1050 if (bic == 0) { 1051 return; 1052 } 1053 s = splvm(); 1054 while (sva < eva) { 1055 if ((ptp = pte_find(pm, sva)) != NULL) { 1056 *ptp &= ~bic; 1057 ppc4xx_tlb_flush(sva, pm->pm_ctx); 1058 } 1059 sva += PAGE_SIZE; 1060 } 1061 splx(s); 1062 } 1063 1064 bool 1065 pmap_check_attr(struct vm_page *pg, u_int mask, int clear) 1066 { 1067 paddr_t pa; 1068 char *attr; 1069 int s, rv; 1070 1071 /* 1072 * First modify bits in cache. 1073 */ 1074 pa = VM_PAGE_TO_PHYS(pg); 1075 attr = pa_to_attr(pa); 1076 if (attr == NULL) 1077 return false; 1078 1079 s = splvm(); 1080 rv = ((*attr & mask) != 0); 1081 if (clear) { 1082 *attr &= ~mask; 1083 pmap_page_protect(pg, mask == PMAP_ATTR_CHG ? VM_PROT_READ : 0); 1084 } 1085 splx(s); 1086 return rv; 1087 } 1088 1089 1090 /* 1091 * Lower the protection on the specified physical page. 1092 * 1093 * There are only two cases: either the protection is going to 0, 1094 * or it is going to read-only. 1095 */ 1096 void 1097 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 1098 { 1099 paddr_t pa = VM_PAGE_TO_PHYS(pg); 1100 vaddr_t va; 1101 struct pv_entry *pvh, *pv, *npv; 1102 struct pmap *pm; 1103 1104 pvh = pa_to_pv(pa); 1105 if (pvh == NULL) 1106 return; 1107 1108 /* Handle extra pvs which may be deleted in the operation */ 1109 for (pv = pvh->pv_next; pv; pv = npv) { 1110 npv = pv->pv_next; 1111 1112 pm = pv->pv_pm; 1113 va = pv->pv_va; 1114 pmap_protect(pm, va, va + PAGE_SIZE, prot); 1115 } 1116 /* Now check the head pv */ 1117 if (pvh->pv_pm) { 1118 pv = pvh; 1119 pm = pv->pv_pm; 1120 va = pv->pv_va; 1121 pmap_protect(pm, va, va + PAGE_SIZE, prot); 1122 } 1123 } 1124 1125 /* 1126 * Activate the address space for the specified process. If the process 1127 * is the current process, load the new MMU context. 1128 */ 1129 void 1130 pmap_activate(struct lwp *l) 1131 { 1132 #if 0 1133 struct pcb *pcb = lwp_getpcb(l); 1134 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 1135 1136 /* 1137 * XXX Normally performed in cpu_lwp_fork(). 1138 */ 1139 printf("pmap_activate(%p), pmap=%p\n",l,pmap); 1140 pcb->pcb_pm = pmap; 1141 #endif 1142 } 1143 1144 /* 1145 * Deactivate the specified process's address space. 1146 */ 1147 void 1148 pmap_deactivate(struct lwp *l) 1149 { 1150 } 1151 1152 /* 1153 * Synchronize caches corresponding to [addr, addr+len) in p. 1154 */ 1155 void 1156 pmap_procwr(struct proc *p, vaddr_t va, size_t len) 1157 { 1158 struct pmap *pm = p->p_vmspace->vm_map.pmap; 1159 int msr, ctx, opid, step; 1160 1161 step = CACHELINESIZE; 1162 1163 /* 1164 * Need to turn off IMMU and switch to user context. 1165 * (icbi uses DMMU). 1166 */ 1167 if (!(ctx = pm->pm_ctx)) { 1168 /* No context -- assign it one */ 1169 ctx_alloc(pm); 1170 ctx = pm->pm_ctx; 1171 } 1172 __asm volatile("mfmsr %0;" 1173 "li %1, %7;" 1174 "andc %1,%0,%1;" 1175 "mtmsr %1;" 1176 "sync;isync;" 1177 "mfpid %1;" 1178 "mtpid %2;" 1179 "sync; isync;" 1180 "1:" 1181 "dcbf 0,%3;" 1182 "icbi 0,%3;" 1183 "add %3,%3,%5;" 1184 "addc. %4,%4,%6;" 1185 "bge 1b;" 1186 "mtpid %1;" 1187 "mtmsr %0;" 1188 "sync; isync" 1189 : "=&r" (msr), "=&r" (opid) 1190 : "r" (ctx), "r" (va), "r" (len), "r" (step), "r" (-step), 1191 "K" (PSL_IR | PSL_DR)); 1192 } 1193 1194 1195 /* This has to be done in real mode !!! */ 1196 void 1197 ppc4xx_tlb_flush(vaddr_t va, int pid) 1198 { 1199 u_long i, found; 1200 u_long msr; 1201 1202 /* If there's no context then it can't be mapped. */ 1203 if (!pid) 1204 return; 1205 1206 __asm( "mfpid %1;" /* Save PID */ 1207 "mfmsr %2;" /* Save MSR */ 1208 "li %0,0;" /* Now clear MSR */ 1209 "mtmsr %0;" 1210 "mtpid %4;" /* Set PID */ 1211 "sync;" 1212 "tlbsx. %0,0,%3;" /* Search TLB */ 1213 "sync;" 1214 "mtpid %1;" /* Restore PID */ 1215 "mtmsr %2;" /* Restore MSR */ 1216 "sync;isync;" 1217 "li %1,1;" 1218 "beq 1f;" 1219 "li %1,0;" 1220 "1:" 1221 : "=&r" (i), "=&r" (found), "=&r" (msr) 1222 : "r" (va), "r" (pid)); 1223 if (found && !TLB_LOCKED(i)) { 1224 1225 /* Now flush translation */ 1226 __asm volatile( 1227 "tlbwe %0,%1,0;" 1228 "sync;isync;" 1229 : : "r" (0), "r" (i)); 1230 1231 tlb_info[i].ti_ctx = 0; 1232 tlb_info[i].ti_flags = 0; 1233 tlbnext = i; 1234 /* Successful flushes */ 1235 tlbflush_ev.ev_count++; 1236 } 1237 } 1238 1239 void 1240 ppc4xx_tlb_flush_all(void) 1241 { 1242 u_long i; 1243 1244 for (i = 0; i < NTLB; i++) 1245 if (!TLB_LOCKED(i)) { 1246 __asm volatile( 1247 "tlbwe %0,%1,0;" 1248 "sync;isync;" 1249 : : "r" (0), "r" (i)); 1250 tlb_info[i].ti_ctx = 0; 1251 tlb_info[i].ti_flags = 0; 1252 } 1253 1254 __asm volatile("sync;isync"); 1255 } 1256 1257 /* Find a TLB entry to evict. */ 1258 static int 1259 ppc4xx_tlb_find_victim(void) 1260 { 1261 int flags; 1262 1263 for (;;) { 1264 if (++tlbnext >= NTLB) 1265 tlbnext = tlb_nreserved; 1266 flags = tlb_info[tlbnext].ti_flags; 1267 if (!(flags & TLBF_USED) || 1268 (flags & (TLBF_LOCKED | TLBF_REF)) == 0) { 1269 u_long va, stack = (u_long)&va; 1270 1271 if (!((tlb_info[tlbnext].ti_va ^ stack) & (~PGOFSET)) && 1272 (tlb_info[tlbnext].ti_ctx == KERNEL_PID) && 1273 (flags & TLBF_USED)) { 1274 /* Kernel stack page */ 1275 flags |= TLBF_USED; 1276 tlb_info[tlbnext].ti_flags = flags; 1277 } else { 1278 /* Found it! */ 1279 return (tlbnext); 1280 } 1281 } else { 1282 tlb_info[tlbnext].ti_flags = (flags & ~TLBF_REF); 1283 } 1284 } 1285 } 1286 1287 void 1288 ppc4xx_tlb_enter(int ctx, vaddr_t va, u_int pte) 1289 { 1290 u_long th, tl, idx; 1291 tlbpid_t pid; 1292 u_short msr; 1293 paddr_t pa; 1294 int sz; 1295 1296 tlbenter_ev.ev_count++; 1297 1298 sz = (pte & TTE_SZ_MASK) >> TTE_SZ_SHIFT; 1299 pa = (pte & TTE_RPN_MASK(sz)); 1300 th = (va & TLB_EPN_MASK) | (sz << TLB_SIZE_SHFT) | TLB_VALID; 1301 tl = (pte & ~TLB_RPN_MASK) | pa; 1302 tl |= ppc4xx_tlbflags(va, pa); 1303 1304 idx = ppc4xx_tlb_find_victim(); 1305 1306 #ifdef DIAGNOSTIC 1307 if ((idx < tlb_nreserved) || (idx >= NTLB)) { 1308 panic("ppc4xx_tlb_enter: replacing entry %ld", idx); 1309 } 1310 #endif 1311 1312 tlb_info[idx].ti_va = (va & TLB_EPN_MASK); 1313 tlb_info[idx].ti_ctx = ctx; 1314 tlb_info[idx].ti_flags = TLBF_USED | TLBF_REF; 1315 1316 __asm volatile( 1317 "mfmsr %0;" /* Save MSR */ 1318 "li %1,0;" 1319 "tlbwe %1,%3,0;" /* Invalidate old entry. */ 1320 "mtmsr %1;" /* Clear MSR */ 1321 "mfpid %1;" /* Save old PID */ 1322 "mtpid %2;" /* Load translation ctx */ 1323 "sync; isync;" 1324 #ifdef DEBUG 1325 "andi. %3,%3,63;" 1326 "tweqi %3,0;" /* XXXXX DEBUG trap on index 0 */ 1327 #endif 1328 "tlbwe %4,%3,1; tlbwe %5,%3,0;" /* Set TLB */ 1329 "sync; isync;" 1330 "mtpid %1; mtmsr %0;" /* Restore PID and MSR */ 1331 "sync; isync;" 1332 : "=&r" (msr), "=&r" (pid) 1333 : "r" (ctx), "r" (idx), "r" (tl), "r" (th)); 1334 } 1335 1336 void 1337 ppc4xx_tlb_init(void) 1338 { 1339 int i; 1340 1341 /* Mark reserved TLB entries */ 1342 for (i = 0; i < tlb_nreserved; i++) { 1343 tlb_info[i].ti_flags = TLBF_LOCKED | TLBF_USED; 1344 tlb_info[i].ti_ctx = KERNEL_PID; 1345 } 1346 1347 /* Setup security zones */ 1348 /* Z0 - accessible by kernel only if TLB entry permissions allow 1349 * Z1,Z2 - access is controlled by TLB entry permissions 1350 * Z3 - full access regardless of TLB entry permissions 1351 */ 1352 1353 __asm volatile( 1354 "mtspr %0,%1;" 1355 "sync;" 1356 :: "K"(SPR_ZPR), "r" (0x1b000000)); 1357 } 1358 1359 /* 1360 * ppc4xx_tlb_size_mask: 1361 * 1362 * Roundup size to supported page size, return TLBHI mask and real size. 1363 */ 1364 static int 1365 ppc4xx_tlb_size_mask(size_t size, int *mask, int *rsiz) 1366 { 1367 int i; 1368 1369 for (i = 0; i < __arraycount(tlbsize); i++) 1370 if (size <= tlbsize[i]) { 1371 *mask = (i << TLB_SIZE_SHFT); 1372 *rsiz = tlbsize[i]; 1373 return (0); 1374 } 1375 return (EINVAL); 1376 } 1377 1378 /* 1379 * ppc4xx_tlb_mapiodev: 1380 * 1381 * Lookup virtual address of mapping previously entered via 1382 * ppc4xx_tlb_reserve. Search TLB directly so that we don't 1383 * need to waste extra storage for reserved mappings. Note 1384 * that reading TLBHI also sets PID, but all reserved mappings 1385 * use KERNEL_PID, so the side effect is nil. 1386 */ 1387 void * 1388 ppc4xx_tlb_mapiodev(paddr_t base, psize_t len) 1389 { 1390 paddr_t pa; 1391 vaddr_t va; 1392 u_int lo, hi, sz; 1393 int i; 1394 1395 /* tlb_nreserved is only allowed to grow, so this is safe. */ 1396 for (i = 0; i < tlb_nreserved; i++) { 1397 __asm volatile ( 1398 " tlbre %0,%2,1 \n" /* TLBLO */ 1399 " tlbre %1,%2,0 \n" /* TLBHI */ 1400 : "=&r" (lo), "=&r" (hi) 1401 : "r" (i)); 1402 1403 KASSERT(hi & TLB_VALID); 1404 KASSERT(mfspr(SPR_PID) == KERNEL_PID); 1405 1406 pa = (lo & TLB_RPN_MASK); 1407 if (base < pa) 1408 continue; 1409 1410 sz = tlbsize[(hi & TLB_SIZE_MASK) >> TLB_SIZE_SHFT]; 1411 if ((base + len) > (pa + sz)) 1412 continue; 1413 1414 va = (hi & TLB_EPN_MASK) + (base & (sz - 1)); /* sz = 2^n */ 1415 return (void *)(va); 1416 } 1417 1418 return (NULL); 1419 } 1420 1421 /* 1422 * ppc4xx_tlb_reserve: 1423 * 1424 * Map physical range to kernel virtual chunk via reserved TLB entry. 1425 */ 1426 void 1427 ppc4xx_tlb_reserve(paddr_t pa, vaddr_t va, size_t size, int flags) 1428 { 1429 u_int lo, hi; 1430 int szmask, rsize; 1431 1432 /* Called before pmap_bootstrap(), va outside kernel space. */ 1433 KASSERT(va < VM_MIN_KERNEL_ADDRESS || va >= VM_MAX_KERNEL_ADDRESS); 1434 KASSERT(! pmap_bootstrap_done); 1435 KASSERT(tlb_nreserved < NTLB); 1436 1437 /* Resolve size. */ 1438 if (ppc4xx_tlb_size_mask(size, &szmask, &rsize) != 0) 1439 panic("ppc4xx_tlb_reserve: entry %d, %zuB too large", 1440 size, tlb_nreserved); 1441 1442 /* Real size will be power of two >= 1024, so this is OK. */ 1443 pa &= ~(rsize - 1); /* RPN */ 1444 va &= ~(rsize - 1); /* EPN */ 1445 1446 lo = pa | TLB_WR | flags; 1447 hi = va | TLB_VALID | szmask; 1448 1449 #ifdef PPC_4XX_NOCACHE 1450 lo |= TLB_I; 1451 #endif 1452 1453 __asm volatile( 1454 " tlbwe %1,%0,1 \n" /* write TLBLO */ 1455 " tlbwe %2,%0,0 \n" /* write TLBHI */ 1456 " sync \n" 1457 " isync \n" 1458 : : "r" (tlb_nreserved), "r" (lo), "r" (hi)); 1459 1460 tlb_nreserved++; 1461 } 1462 1463 /* 1464 * We should pass the ctx in from trap code. 1465 */ 1466 int 1467 pmap_tlbmiss(vaddr_t va, int ctx) 1468 { 1469 volatile u_int *pte; 1470 u_long tte; 1471 1472 tlbmiss_ev.ev_count++; 1473 1474 /* 1475 * We will reserve 0 upto VM_MIN_KERNEL_ADDRESS for va == pa mappings. 1476 * Physical RAM is expected to live in this range, care must be taken 1477 * to not clobber 0 upto ${physmem} with device mappings in machdep 1478 * code. 1479 */ 1480 if (ctx != KERNEL_PID || 1481 (va >= VM_MIN_KERNEL_ADDRESS && va < VM_MAX_KERNEL_ADDRESS)) { 1482 pte = pte_find((struct pmap *)__UNVOLATILE(ctxbusy[ctx]), va); 1483 if (pte == NULL) { 1484 /* Map unmanaged addresses directly for kernel access */ 1485 return 1; 1486 } 1487 tte = *pte; 1488 if (tte == 0) { 1489 return 1; 1490 } 1491 } else { 1492 /* Create a 16MB writable mapping. */ 1493 #ifdef PPC_4XX_NOCACHE 1494 tte = TTE_PA(va) | TTE_ZONE(ZONE_PRIV) | TTE_SZ_16M | TTE_I |TTE_WR; 1495 #else 1496 tte = TTE_PA(va) | TTE_ZONE(ZONE_PRIV) | TTE_SZ_16M | TTE_WR; 1497 #endif 1498 } 1499 tlbhit_ev.ev_count++; 1500 ppc4xx_tlb_enter(ctx, va, tte); 1501 1502 return 0; 1503 } 1504 1505 /* 1506 * Flush all the entries matching a context from the TLB. 1507 */ 1508 static int 1509 ctx_flush(int cnum) 1510 { 1511 int i; 1512 1513 /* We gotta steal this context */ 1514 for (i = tlb_nreserved; i < NTLB; i++) { 1515 if (tlb_info[i].ti_ctx == cnum) { 1516 /* Can't steal ctx if it has a locked entry. */ 1517 if (TLB_LOCKED(i)) { 1518 #ifdef DIAGNOSTIC 1519 printf("ctx_flush: can't invalidate " 1520 "locked mapping %d " 1521 "for context %d\n", i, cnum); 1522 #ifdef DDB 1523 Debugger(); 1524 #endif 1525 #endif 1526 return (1); 1527 } 1528 #ifdef DIAGNOSTIC 1529 if (i < tlb_nreserved) 1530 panic("TLB entry %d not locked", i); 1531 #endif 1532 /* Invalidate particular TLB entry regardless of locked status */ 1533 __asm volatile("tlbwe %0,%1,0" : :"r"(0),"r"(i)); 1534 tlb_info[i].ti_flags = 0; 1535 } 1536 } 1537 return (0); 1538 } 1539 1540 /* 1541 * Allocate a context. If necessary, steal one from someone else. 1542 * 1543 * The new context is flushed from the TLB before returning. 1544 */ 1545 int 1546 ctx_alloc(struct pmap *pm) 1547 { 1548 int s, cnum; 1549 static int next = MINCTX; 1550 1551 if (pm == pmap_kernel()) { 1552 #ifdef DIAGNOSTIC 1553 printf("ctx_alloc: kernel pmap!\n"); 1554 #endif 1555 return (0); 1556 } 1557 s = splvm(); 1558 1559 /* Find a likely context. */ 1560 cnum = next; 1561 do { 1562 if ((++cnum) > NUMCTX) 1563 cnum = MINCTX; 1564 } while (ctxbusy[cnum] != NULL && cnum != next); 1565 1566 /* Now clean it out */ 1567 oops: 1568 if (cnum < MINCTX) 1569 cnum = MINCTX; /* Never steal ctx 0 or 1 */ 1570 if (ctx_flush(cnum)) { 1571 /* oops -- something's wired. */ 1572 if ((++cnum) > NUMCTX) 1573 cnum = MINCTX; 1574 goto oops; 1575 } 1576 1577 if (ctxbusy[cnum]) { 1578 #ifdef DEBUG 1579 /* We should identify this pmap and clear it */ 1580 printf("Warning: stealing context %d\n", cnum); 1581 #endif 1582 ctxbusy[cnum]->pm_ctx = 0; 1583 } 1584 ctxbusy[cnum] = pm; 1585 next = cnum; 1586 splx(s); 1587 pm->pm_ctx = cnum; 1588 1589 return cnum; 1590 } 1591 1592 /* 1593 * Give away a context. 1594 */ 1595 void 1596 ctx_free(struct pmap *pm) 1597 { 1598 int oldctx; 1599 1600 oldctx = pm->pm_ctx; 1601 1602 if (oldctx == 0) 1603 panic("ctx_free: freeing kernel context"); 1604 #ifdef DIAGNOSTIC 1605 if (ctxbusy[oldctx] == 0) 1606 printf("ctx_free: freeing free context %d\n", oldctx); 1607 if (ctxbusy[oldctx] != pm) { 1608 printf("ctx_free: freeing someone esle's context\n " 1609 "ctxbusy[%d] = %p, pm->pm_ctx = %p\n", 1610 oldctx, (void *)(u_long)ctxbusy[oldctx], pm); 1611 #ifdef DDB 1612 Debugger(); 1613 #endif 1614 } 1615 #endif 1616 /* We should verify it has not been stolen and reallocated... */ 1617 ctxbusy[oldctx] = NULL; 1618 ctx_flush(oldctx); 1619 } 1620 1621 1622 #ifdef DEBUG 1623 /* 1624 * Test ref/modify handling. 1625 */ 1626 void pmap_testout(void); 1627 void 1628 pmap_testout(void) 1629 { 1630 vaddr_t va; 1631 volatile int *loc; 1632 int val = 0; 1633 paddr_t pa; 1634 struct vm_page *pg; 1635 int ref, mod; 1636 1637 /* Allocate a page */ 1638 va = (vaddr_t)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 1639 UVM_KMF_WIRED | UVM_KMF_ZERO); 1640 loc = (int*)va; 1641 1642 pmap_extract(pmap_kernel(), va, &pa); 1643 pg = PHYS_TO_VM_PAGE(pa); 1644 pmap_unwire(pmap_kernel(), va); 1645 1646 pmap_kremove(va, PAGE_SIZE); 1647 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1648 pmap_update(pmap_kernel()); 1649 1650 /* Now clear reference and modify */ 1651 ref = pmap_clear_reference(pg); 1652 mod = pmap_clear_modify(pg); 1653 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1654 (void *)(u_long)va, (long)pa, 1655 ref, mod); 1656 1657 /* Check it's properly cleared */ 1658 ref = pmap_is_referenced(pg); 1659 mod = pmap_is_modified(pg); 1660 printf("Checking cleared page: ref %d, mod %d\n", 1661 ref, mod); 1662 1663 /* Reference page */ 1664 val = *loc; 1665 1666 ref = pmap_is_referenced(pg); 1667 mod = pmap_is_modified(pg); 1668 printf("Referenced page: ref %d, mod %d val %x\n", 1669 ref, mod, val); 1670 1671 /* Now clear reference and modify */ 1672 ref = pmap_clear_reference(pg); 1673 mod = pmap_clear_modify(pg); 1674 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1675 (void *)(u_long)va, (long)pa, 1676 ref, mod); 1677 1678 /* Modify page */ 1679 *loc = 1; 1680 1681 ref = pmap_is_referenced(pg); 1682 mod = pmap_is_modified(pg); 1683 printf("Modified page: ref %d, mod %d\n", 1684 ref, mod); 1685 1686 /* Now clear reference and modify */ 1687 ref = pmap_clear_reference(pg); 1688 mod = pmap_clear_modify(pg); 1689 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1690 (void *)(u_long)va, (long)pa, 1691 ref, mod); 1692 1693 /* Check it's properly cleared */ 1694 ref = pmap_is_referenced(pg); 1695 mod = pmap_is_modified(pg); 1696 printf("Checking cleared page: ref %d, mod %d\n", 1697 ref, mod); 1698 1699 /* Modify page */ 1700 *loc = 1; 1701 1702 ref = pmap_is_referenced(pg); 1703 mod = pmap_is_modified(pg); 1704 printf("Modified page: ref %d, mod %d\n", 1705 ref, mod); 1706 1707 /* Check pmap_protect() */ 1708 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_READ); 1709 pmap_update(pmap_kernel()); 1710 ref = pmap_is_referenced(pg); 1711 mod = pmap_is_modified(pg); 1712 printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n", 1713 ref, mod); 1714 1715 /* Now clear reference and modify */ 1716 ref = pmap_clear_reference(pg); 1717 mod = pmap_clear_modify(pg); 1718 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1719 (void *)(u_long)va, (long)pa, 1720 ref, mod); 1721 1722 /* Reference page */ 1723 val = *loc; 1724 1725 ref = pmap_is_referenced(pg); 1726 mod = pmap_is_modified(pg); 1727 printf("Referenced page: ref %d, mod %d val %x\n", 1728 ref, mod, val); 1729 1730 /* Now clear reference and modify */ 1731 ref = pmap_clear_reference(pg); 1732 mod = pmap_clear_modify(pg); 1733 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1734 (void *)(u_long)va, (long)pa, 1735 ref, mod); 1736 1737 /* Modify page */ 1738 #if 0 1739 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1740 pmap_update(pmap_kernel()); 1741 #endif 1742 *loc = 1; 1743 1744 ref = pmap_is_referenced(pg); 1745 mod = pmap_is_modified(pg); 1746 printf("Modified page: ref %d, mod %d\n", 1747 ref, mod); 1748 1749 /* Check pmap_protect() */ 1750 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_NONE); 1751 pmap_update(pmap_kernel()); 1752 ref = pmap_is_referenced(pg); 1753 mod = pmap_is_modified(pg); 1754 printf("pmap_protect(): ref %d, mod %d\n", 1755 ref, mod); 1756 1757 /* Now clear reference and modify */ 1758 ref = pmap_clear_reference(pg); 1759 mod = pmap_clear_modify(pg); 1760 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1761 (void *)(u_long)va, (long)pa, 1762 ref, mod); 1763 1764 /* Reference page */ 1765 val = *loc; 1766 1767 ref = pmap_is_referenced(pg); 1768 mod = pmap_is_modified(pg); 1769 printf("Referenced page: ref %d, mod %d val %x\n", 1770 ref, mod, val); 1771 1772 /* Now clear reference and modify */ 1773 ref = pmap_clear_reference(pg); 1774 mod = pmap_clear_modify(pg); 1775 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1776 (void *)(u_long)va, (long)pa, 1777 ref, mod); 1778 1779 /* Modify page */ 1780 #if 0 1781 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1782 pmap_update(pmap_kernel()); 1783 #endif 1784 *loc = 1; 1785 1786 ref = pmap_is_referenced(pg); 1787 mod = pmap_is_modified(pg); 1788 printf("Modified page: ref %d, mod %d\n", 1789 ref, mod); 1790 1791 /* Check pmap_pag_protect() */ 1792 pmap_page_protect(pg, VM_PROT_READ); 1793 ref = pmap_is_referenced(pg); 1794 mod = pmap_is_modified(pg); 1795 printf("pmap_page_protect(VM_PROT_READ): ref %d, mod %d\n", 1796 ref, mod); 1797 1798 /* Now clear reference and modify */ 1799 ref = pmap_clear_reference(pg); 1800 mod = pmap_clear_modify(pg); 1801 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1802 (void *)(u_long)va, (long)pa, 1803 ref, mod); 1804 1805 /* Reference page */ 1806 val = *loc; 1807 1808 ref = pmap_is_referenced(pg); 1809 mod = pmap_is_modified(pg); 1810 printf("Referenced page: ref %d, mod %d val %x\n", 1811 ref, mod, val); 1812 1813 /* Now clear reference and modify */ 1814 ref = pmap_clear_reference(pg); 1815 mod = pmap_clear_modify(pg); 1816 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1817 (void *)(u_long)va, (long)pa, 1818 ref, mod); 1819 1820 /* Modify page */ 1821 #if 0 1822 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1823 pmap_update(pmap_kernel()); 1824 #endif 1825 *loc = 1; 1826 1827 ref = pmap_is_referenced(pg); 1828 mod = pmap_is_modified(pg); 1829 printf("Modified page: ref %d, mod %d\n", 1830 ref, mod); 1831 1832 /* Check pmap_pag_protect() */ 1833 pmap_page_protect(pg, VM_PROT_NONE); 1834 ref = pmap_is_referenced(pg); 1835 mod = pmap_is_modified(pg); 1836 printf("pmap_page_protect(): ref %d, mod %d\n", 1837 ref, mod); 1838 1839 /* Now clear reference and modify */ 1840 ref = pmap_clear_reference(pg); 1841 mod = pmap_clear_modify(pg); 1842 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1843 (void *)(u_long)va, (long)pa, 1844 ref, mod); 1845 1846 1847 /* Reference page */ 1848 val = *loc; 1849 1850 ref = pmap_is_referenced(pg); 1851 mod = pmap_is_modified(pg); 1852 printf("Referenced page: ref %d, mod %d val %x\n", 1853 ref, mod, val); 1854 1855 /* Now clear reference and modify */ 1856 ref = pmap_clear_reference(pg); 1857 mod = pmap_clear_modify(pg); 1858 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1859 (void *)(u_long)va, (long)pa, 1860 ref, mod); 1861 1862 /* Modify page */ 1863 #if 0 1864 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1865 pmap_update(pmap_kernel()); 1866 #endif 1867 *loc = 1; 1868 1869 ref = pmap_is_referenced(pg); 1870 mod = pmap_is_modified(pg); 1871 printf("Modified page: ref %d, mod %d\n", 1872 ref, mod); 1873 1874 /* Unmap page */ 1875 pmap_remove(pmap_kernel(), va, va+1); 1876 pmap_update(pmap_kernel()); 1877 ref = pmap_is_referenced(pg); 1878 mod = pmap_is_modified(pg); 1879 printf("Unmapped page: ref %d, mod %d\n", ref, mod); 1880 1881 /* Now clear reference and modify */ 1882 ref = pmap_clear_reference(pg); 1883 mod = pmap_clear_modify(pg); 1884 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1885 (void *)(u_long)va, (long)pa, ref, mod); 1886 1887 /* Check it's properly cleared */ 1888 ref = pmap_is_referenced(pg); 1889 mod = pmap_is_modified(pg); 1890 printf("Checking cleared page: ref %d, mod %d\n", 1891 ref, mod); 1892 1893 pmap_remove(pmap_kernel(), va, va + PAGE_SIZE); 1894 pmap_kenter_pa(va, pa, VM_PROT_ALL, 0); 1895 uvm_km_free(kernel_map, (vaddr_t)va, PAGE_SIZE, UVM_KMF_WIRED); 1896 } 1897 #endif 1898