1 /* $NetBSD: pmap.c,v 1.67 2011/06/18 06:41:41 matt Exp $ */ 2 3 /* 4 * Copyright 2001 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Eduardo Horvath and Simon Burge for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 40 * Copyright (C) 1995, 1996 TooLs GmbH. 41 * All rights reserved. 42 * 43 * Redistribution and use in source and binary forms, with or without 44 * modification, are permitted provided that the following conditions 45 * are met: 46 * 1. Redistributions of source code must retain the above copyright 47 * notice, this list of conditions and the following disclaimer. 48 * 2. Redistributions in binary form must reproduce the above copyright 49 * notice, this list of conditions and the following disclaimer in the 50 * documentation and/or other materials provided with the distribution. 51 * 3. All advertising materials mentioning features or use of this software 52 * must display the following acknowledgement: 53 * This product includes software developed by TooLs GmbH. 54 * 4. The name of TooLs GmbH may not be used to endorse or promote products 55 * derived from this software without specific prior written permission. 56 * 57 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 58 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 59 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 60 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 61 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 62 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 63 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 64 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 65 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 66 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 67 */ 68 69 #include <sys/cdefs.h> 70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.67 2011/06/18 06:41:41 matt Exp $"); 71 72 #include <sys/param.h> 73 #include <sys/malloc.h> 74 #include <sys/proc.h> 75 #include <sys/queue.h> 76 #include <sys/systm.h> 77 #include <sys/pool.h> 78 #include <sys/device.h> 79 #include <sys/cpu.h> 80 81 #include <uvm/uvm.h> 82 83 #include <machine/pcb.h> 84 #include <machine/powerpc.h> 85 #include <machine/tlb.h> 86 87 #include <powerpc/spr.h> 88 #include <powerpc/ibm4xx/spr.h> 89 90 #include <powerpc/ibm4xx/cpu.h> 91 92 /* 93 * kernmap is an array of PTEs large enough to map in 94 * 4GB. At 16KB/page it is 256K entries or 2MB. 95 */ 96 #define KERNMAP_SIZE ((0xffffffffU/PAGE_SIZE)+1) 97 void *kernmap; 98 99 #define MINCTX 2 100 #define NUMCTX 256 101 102 volatile struct pmap *ctxbusy[NUMCTX]; 103 104 #define TLBF_USED 0x1 105 #define TLBF_REF 0x2 106 #define TLBF_LOCKED 0x4 107 #define TLB_LOCKED(i) (tlb_info[(i)].ti_flags & TLBF_LOCKED) 108 109 typedef struct tlb_info_s { 110 char ti_flags; 111 char ti_ctx; /* TLB_PID assiciated with the entry */ 112 u_int ti_va; 113 } tlb_info_t; 114 115 volatile tlb_info_t tlb_info[NTLB]; 116 /* We'll use a modified FIFO replacement policy cause it's cheap */ 117 volatile int tlbnext; 118 119 static int tlb_nreserved = 0; 120 static int pmap_bootstrap_done = 0; 121 122 /* Event counters */ 123 struct evcnt tlbmiss_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 124 NULL, "cpu", "tlbmiss"); 125 struct evcnt tlbhit_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 126 NULL, "cpu", "tlbhit"); 127 struct evcnt tlbflush_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 128 NULL, "cpu", "tlbflush"); 129 struct evcnt tlbenter_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 130 NULL, "cpu", "tlbenter"); 131 EVCNT_ATTACH_STATIC(tlbmiss_ev); 132 EVCNT_ATTACH_STATIC(tlbhit_ev); 133 EVCNT_ATTACH_STATIC(tlbflush_ev); 134 EVCNT_ATTACH_STATIC(tlbenter_ev); 135 136 struct pmap kernel_pmap_; 137 struct pmap *const kernel_pmap_ptr = &kernel_pmap_; 138 139 static int npgs; 140 static u_int nextavail; 141 #ifndef MSGBUFADDR 142 extern paddr_t msgbuf_paddr; 143 #endif 144 145 static struct mem_region *mem, *avail; 146 147 /* 148 * This is a cache of referenced/modified bits. 149 * Bits herein are shifted by ATTRSHFT. 150 */ 151 static char *pmap_attrib; 152 153 #define PV_WIRED 0x1 154 #define PV_WIRE(pv) ((pv)->pv_va |= PV_WIRED) 155 #define PV_UNWIRE(pv) ((pv)->pv_va &= ~PV_WIRED) 156 #define PV_ISWIRED(pv) ((pv)->pv_va & PV_WIRED) 157 #define PV_CMPVA(va,pv) (!(((pv)->pv_va ^ (va)) & (~PV_WIRED))) 158 159 struct pv_entry { 160 struct pv_entry *pv_next; /* Linked list of mappings */ 161 vaddr_t pv_va; /* virtual address of mapping */ 162 struct pmap *pv_pm; 163 }; 164 165 /* Each index corresponds to TLB_SIZE_* value. */ 166 static size_t tlbsize[] = { 167 1024, /* TLB_SIZE_1K */ 168 4096, /* TLB_SIZE_4K */ 169 16384, /* TLB_SIZE_16K */ 170 65536, /* TLB_SIZE_64K */ 171 262144, /* TLB_SIZE_256K */ 172 1048576, /* TLB_SIZE_1M */ 173 4194304, /* TLB_SIZE_4M */ 174 16777216, /* TLB_SIZE_16M */ 175 }; 176 177 struct pv_entry *pv_table; 178 static struct pool pv_pool; 179 180 static int pmap_initialized; 181 182 static int ctx_flush(int); 183 184 inline struct pv_entry *pa_to_pv(paddr_t); 185 static inline char *pa_to_attr(paddr_t); 186 187 static inline volatile u_int *pte_find(struct pmap *, vaddr_t); 188 static inline int pte_enter(struct pmap *, vaddr_t, u_int); 189 190 static inline int pmap_enter_pv(struct pmap *, vaddr_t, paddr_t, int); 191 static void pmap_remove_pv(struct pmap *, vaddr_t, paddr_t); 192 193 static int ppc4xx_tlb_size_mask(size_t, int *, int *); 194 195 196 inline struct pv_entry * 197 pa_to_pv(paddr_t pa) 198 { 199 int bank, pg; 200 201 bank = vm_physseg_find(atop(pa), &pg); 202 if (bank == -1) 203 return NULL; 204 return &VM_PHYSMEM_PTR(bank)->pmseg.pvent[pg]; 205 } 206 207 static inline char * 208 pa_to_attr(paddr_t pa) 209 { 210 int bank, pg; 211 212 bank = vm_physseg_find(atop(pa), &pg); 213 if (bank == -1) 214 return NULL; 215 return &VM_PHYSMEM_PTR(bank)->pmseg.attrs[pg]; 216 } 217 218 /* 219 * Insert PTE into page table. 220 */ 221 int 222 pte_enter(struct pmap *pm, vaddr_t va, u_int pte) 223 { 224 int seg = STIDX(va); 225 int ptn = PTIDX(va); 226 u_int oldpte; 227 228 if (!pm->pm_ptbl[seg]) { 229 /* Don't allocate a page to clear a non-existent mapping. */ 230 if (!pte) 231 return (0); 232 /* Allocate a page XXXX this will sleep! */ 233 pm->pm_ptbl[seg] = 234 (uint *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 235 UVM_KMF_WIRED | UVM_KMF_ZERO); 236 } 237 oldpte = pm->pm_ptbl[seg][ptn]; 238 pm->pm_ptbl[seg][ptn] = pte; 239 240 /* Flush entry. */ 241 ppc4xx_tlb_flush(va, pm->pm_ctx); 242 if (oldpte != pte) { 243 if (pte == 0) 244 pm->pm_stats.resident_count--; 245 else 246 pm->pm_stats.resident_count++; 247 } 248 return (1); 249 } 250 251 /* 252 * Get a pointer to a PTE in a page table. 253 */ 254 volatile u_int * 255 pte_find(struct pmap *pm, vaddr_t va) 256 { 257 int seg = STIDX(va); 258 int ptn = PTIDX(va); 259 260 if (pm->pm_ptbl[seg]) 261 return (&pm->pm_ptbl[seg][ptn]); 262 263 return (NULL); 264 } 265 266 /* 267 * This is called during initppc, before the system is really initialized. 268 */ 269 void 270 pmap_bootstrap(u_int kernelstart, u_int kernelend) 271 { 272 struct mem_region *mp, *mp1; 273 int cnt, i; 274 u_int s, e, sz; 275 276 tlbnext = tlb_nreserved; 277 278 /* 279 * Allocate the kernel page table at the end of 280 * kernel space so it's in the locked TTE. 281 */ 282 kernmap = (void *)kernelend; 283 284 /* 285 * Initialize kernel page table. 286 */ 287 for (i = 0; i < STSZ; i++) { 288 pmap_kernel()->pm_ptbl[i] = 0; 289 } 290 ctxbusy[0] = ctxbusy[1] = pmap_kernel(); 291 292 /* 293 * Announce page-size to the VM-system 294 */ 295 uvmexp.pagesize = NBPG; 296 uvm_setpagesize(); 297 298 /* 299 * Get memory. 300 */ 301 mem_regions(&mem, &avail); 302 for (mp = mem; mp->size; mp++) { 303 physmem += btoc(mp->size); 304 printf("+%lx,",mp->size); 305 } 306 printf("\n"); 307 ppc4xx_tlb_init(); 308 /* 309 * Count the number of available entries. 310 */ 311 for (cnt = 0, mp = avail; mp->size; mp++) 312 cnt++; 313 314 /* 315 * Page align all regions. 316 * Non-page aligned memory isn't very interesting to us. 317 * Also, sort the entries for ascending addresses. 318 */ 319 kernelstart &= ~PGOFSET; 320 kernelend = (kernelend + PGOFSET) & ~PGOFSET; 321 for (mp = avail; mp->size; mp++) { 322 s = mp->start; 323 e = mp->start + mp->size; 324 printf("%08x-%08x -> ",s,e); 325 /* 326 * Check whether this region holds all of the kernel. 327 */ 328 if (s < kernelstart && e > kernelend) { 329 avail[cnt].start = kernelend; 330 avail[cnt++].size = e - kernelend; 331 e = kernelstart; 332 } 333 /* 334 * Look whether this regions starts within the kernel. 335 */ 336 if (s >= kernelstart && s < kernelend) { 337 if (e <= kernelend) 338 goto empty; 339 s = kernelend; 340 } 341 /* 342 * Now look whether this region ends within the kernel. 343 */ 344 if (e > kernelstart && e <= kernelend) { 345 if (s >= kernelstart) 346 goto empty; 347 e = kernelstart; 348 } 349 /* 350 * Now page align the start and size of the region. 351 */ 352 s = round_page(s); 353 e = trunc_page(e); 354 if (e < s) 355 e = s; 356 sz = e - s; 357 printf("%08x-%08x = %x\n",s,e,sz); 358 /* 359 * Check whether some memory is left here. 360 */ 361 if (sz == 0) { 362 empty: 363 memmove(mp, mp + 1, 364 (cnt - (mp - avail)) * sizeof *mp); 365 cnt--; 366 mp--; 367 continue; 368 } 369 /* 370 * Do an insertion sort. 371 */ 372 npgs += btoc(sz); 373 for (mp1 = avail; mp1 < mp; mp1++) 374 if (s < mp1->start) 375 break; 376 if (mp1 < mp) { 377 memmove(mp1 + 1, mp1, (char *)mp - (char *)mp1); 378 mp1->start = s; 379 mp1->size = sz; 380 } else { 381 mp->start = s; 382 mp->size = sz; 383 } 384 } 385 386 /* 387 * We cannot do pmap_steal_memory here, 388 * since we don't run with translation enabled yet. 389 */ 390 #ifndef MSGBUFADDR 391 /* 392 * allow for msgbuf 393 */ 394 sz = round_page(MSGBUFSIZE); 395 mp = NULL; 396 for (mp1 = avail; mp1->size; mp1++) 397 if (mp1->size >= sz) 398 mp = mp1; 399 if (mp == NULL) 400 panic("not enough memory?"); 401 402 npgs -= btoc(sz); 403 msgbuf_paddr = mp->start + mp->size - sz; 404 mp->size -= sz; 405 if (mp->size <= 0) 406 memmove(mp, mp + 1, (cnt - (mp - avail)) * sizeof *mp); 407 #endif 408 409 for (mp = avail; mp->size; mp++) 410 uvm_page_physload(atop(mp->start), atop(mp->start + mp->size), 411 atop(mp->start), atop(mp->start + mp->size), 412 VM_FREELIST_DEFAULT); 413 414 /* 415 * Initialize kernel pmap and hardware. 416 */ 417 /* Setup TLB pid allocator so it knows we alreadu using PID 1 */ 418 pmap_kernel()->pm_ctx = KERNEL_PID; 419 nextavail = avail->start; 420 421 pmap_bootstrap_done = 1; 422 } 423 424 /* 425 * Restrict given range to physical memory 426 * 427 * (Used by /dev/mem) 428 */ 429 void 430 pmap_real_memory(paddr_t *start, psize_t *size) 431 { 432 struct mem_region *mp; 433 434 for (mp = mem; mp->size; mp++) { 435 if (*start + *size > mp->start && 436 *start < mp->start + mp->size) { 437 if (*start < mp->start) { 438 *size -= mp->start - *start; 439 *start = mp->start; 440 } 441 if (*start + *size > mp->start + mp->size) 442 *size = mp->start + mp->size - *start; 443 return; 444 } 445 } 446 *size = 0; 447 } 448 449 /* 450 * Initialize anything else for pmap handling. 451 * Called during vm_init(). 452 */ 453 void 454 pmap_init(void) 455 { 456 struct pv_entry *pv; 457 vsize_t sz; 458 vaddr_t addr; 459 int i, s; 460 int bank; 461 char *attr; 462 463 sz = (vsize_t)((sizeof(struct pv_entry) + 1) * npgs); 464 sz = round_page(sz); 465 addr = uvm_km_alloc(kernel_map, sz, 0, UVM_KMF_WIRED | UVM_KMF_ZERO); 466 s = splvm(); 467 pv = pv_table = (struct pv_entry *)addr; 468 for (i = npgs; --i >= 0;) 469 pv++->pv_pm = NULL; 470 pmap_attrib = (char *)pv; 471 memset(pv, 0, npgs); 472 473 pv = pv_table; 474 attr = pmap_attrib; 475 for (bank = 0; bank < vm_nphysseg; bank++) { 476 sz = VM_PHYSMEM_PTR(bank)->end - VM_PHYSMEM_PTR(bank)->start; 477 VM_PHYSMEM_PTR(bank)->pmseg.pvent = pv; 478 VM_PHYSMEM_PTR(bank)->pmseg.attrs = attr; 479 pv += sz; 480 attr += sz; 481 } 482 483 pmap_initialized = 1; 484 splx(s); 485 486 /* Setup a pool for additional pvlist structures */ 487 pool_init(&pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pv_entry", NULL, 488 IPL_VM); 489 } 490 491 /* 492 * How much virtual space is available to the kernel? 493 */ 494 void 495 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 496 { 497 498 #if 0 499 /* 500 * Reserve one segment for kernel virtual memory 501 */ 502 *start = (vaddr_t)(KERNEL_SR << ADDR_SR_SHFT); 503 *end = *start + SEGMENT_LENGTH; 504 #else 505 *start = (vaddr_t) VM_MIN_KERNEL_ADDRESS; 506 *end = (vaddr_t) VM_MAX_KERNEL_ADDRESS; 507 #endif 508 } 509 510 #ifdef PMAP_GROWKERNEL 511 /* 512 * Preallocate kernel page tables to a specified VA. 513 * This simply loops through the first TTE for each 514 * page table from the beginning of the kernel pmap, 515 * reads the entry, and if the result is 516 * zero (either invalid entry or no page table) it stores 517 * a zero there, populating page tables in the process. 518 * This is not the most efficient technique but i don't 519 * expect it to be called that often. 520 */ 521 extern struct vm_page *vm_page_alloc1(void); 522 extern void vm_page_free1(struct vm_page *); 523 524 vaddr_t kbreak = VM_MIN_KERNEL_ADDRESS; 525 526 vaddr_t 527 pmap_growkernel(vaddr_t maxkvaddr) 528 { 529 int s; 530 int seg; 531 paddr_t pg; 532 struct pmap *pm = pmap_kernel(); 533 534 s = splvm(); 535 536 /* Align with the start of a page table */ 537 for (kbreak &= ~(PTMAP-1); kbreak < maxkvaddr; 538 kbreak += PTMAP) { 539 seg = STIDX(kbreak); 540 541 if (pte_find(pm, kbreak)) 542 continue; 543 544 if (uvm.page_init_done) { 545 pg = (paddr_t)VM_PAGE_TO_PHYS(vm_page_alloc1()); 546 } else { 547 if (!uvm_page_physget(&pg)) 548 panic("pmap_growkernel: no memory"); 549 } 550 if (!pg) 551 panic("pmap_growkernel: no pages"); 552 pmap_zero_page((paddr_t)pg); 553 554 /* XXX This is based on all phymem being addressable */ 555 pm->pm_ptbl[seg] = (u_int *)pg; 556 } 557 splx(s); 558 return (kbreak); 559 } 560 561 /* 562 * vm_page_alloc1: 563 * 564 * Allocate and return a memory cell with no associated object. 565 */ 566 struct vm_page * 567 vm_page_alloc1(void) 568 { 569 struct vm_page *pg; 570 571 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE); 572 if (pg) { 573 pg->wire_count = 1; /* no mappings yet */ 574 pg->flags &= ~PG_BUSY; /* never busy */ 575 } 576 return pg; 577 } 578 579 /* 580 * vm_page_free1: 581 * 582 * Returns the given page to the free list, 583 * disassociating it with any VM object. 584 * 585 * Object and page must be locked prior to entry. 586 */ 587 void 588 vm_page_free1(struct vm_page *pg) 589 { 590 #ifdef DIAGNOSTIC 591 if (pg->flags != (PG_CLEAN|PG_FAKE)) { 592 printf("Freeing invalid page %p\n", pg); 593 printf("pa = %llx\n", (unsigned long long)VM_PAGE_TO_PHYS(pg)); 594 #ifdef DDB 595 Debugger(); 596 #endif 597 return; 598 } 599 #endif 600 pg->flags |= PG_BUSY; 601 pg->wire_count = 0; 602 uvm_pagefree(pg); 603 } 604 #endif 605 606 /* 607 * Create and return a physical map. 608 */ 609 struct pmap * 610 pmap_create(void) 611 { 612 struct pmap *pm; 613 614 pm = malloc(sizeof *pm, M_VMPMAP, M_WAITOK); 615 memset(pm, 0, sizeof *pm); 616 pm->pm_refs = 1; 617 return pm; 618 } 619 620 /* 621 * Add a reference to the given pmap. 622 */ 623 void 624 pmap_reference(struct pmap *pm) 625 { 626 627 pm->pm_refs++; 628 } 629 630 /* 631 * Retire the given pmap from service. 632 * Should only be called if the map contains no valid mappings. 633 */ 634 void 635 pmap_destroy(struct pmap *pm) 636 { 637 int i; 638 639 if (--pm->pm_refs > 0) { 640 return; 641 } 642 KASSERT(pm->pm_stats.resident_count == 0); 643 KASSERT(pm->pm_stats.wired_count == 0); 644 for (i = 0; i < STSZ; i++) 645 if (pm->pm_ptbl[i]) { 646 uvm_km_free(kernel_map, (vaddr_t)pm->pm_ptbl[i], 647 PAGE_SIZE, UVM_KMF_WIRED); 648 pm->pm_ptbl[i] = NULL; 649 } 650 if (pm->pm_ctx) 651 ctx_free(pm); 652 free(pm, M_VMPMAP); 653 } 654 655 /* 656 * Copy the range specified by src_addr/len 657 * from the source map to the range dst_addr/len 658 * in the destination map. 659 * 660 * This routine is only advisory and need not do anything. 661 */ 662 void 663 pmap_copy(struct pmap *dst_pmap, struct pmap *src_pmap, vaddr_t dst_addr, 664 vsize_t len, vaddr_t src_addr) 665 { 666 } 667 668 /* 669 * Require that all active physical maps contain no 670 * incorrect entries NOW. 671 */ 672 void 673 pmap_update(struct pmap *pmap) 674 { 675 } 676 677 /* 678 * Fill the given physical page with zeroes. 679 */ 680 void 681 pmap_zero_page(paddr_t pa) 682 { 683 684 #ifdef PPC_4XX_NOCACHE 685 memset((void *)pa, 0, PAGE_SIZE); 686 #else 687 int i; 688 689 for (i = PAGE_SIZE/CACHELINESIZE; i > 0; i--) { 690 __asm volatile ("dcbz 0,%0" :: "r"(pa)); 691 pa += CACHELINESIZE; 692 } 693 #endif 694 } 695 696 /* 697 * Copy the given physical source page to its destination. 698 */ 699 void 700 pmap_copy_page(paddr_t src, paddr_t dst) 701 { 702 703 memcpy((void *)dst, (void *)src, PAGE_SIZE); 704 dcache_flush_page(dst); 705 } 706 707 /* 708 * This returns != 0 on success. 709 */ 710 static inline int 711 pmap_enter_pv(struct pmap *pm, vaddr_t va, paddr_t pa, int flags) 712 { 713 struct pv_entry *pv, *npv = NULL; 714 int s; 715 716 if (!pmap_initialized) 717 return 0; 718 719 s = splvm(); 720 pv = pa_to_pv(pa); 721 if (!pv->pv_pm) { 722 /* 723 * No entries yet, use header as the first entry. 724 */ 725 pv->pv_va = va; 726 pv->pv_pm = pm; 727 pv->pv_next = NULL; 728 } else { 729 /* 730 * There is at least one other VA mapping this page. 731 * Place this entry after the header. 732 */ 733 npv = pool_get(&pv_pool, PR_NOWAIT); 734 if (npv == NULL) { 735 if ((flags & PMAP_CANFAIL) == 0) 736 panic("pmap_enter_pv: failed"); 737 splx(s); 738 return 0; 739 } 740 npv->pv_va = va; 741 npv->pv_pm = pm; 742 npv->pv_next = pv->pv_next; 743 pv->pv_next = npv; 744 pv = npv; 745 } 746 if (flags & PMAP_WIRED) { 747 PV_WIRE(pv); 748 pm->pm_stats.wired_count++; 749 } 750 splx(s); 751 return (1); 752 } 753 754 static void 755 pmap_remove_pv(struct pmap *pm, vaddr_t va, paddr_t pa) 756 { 757 struct pv_entry *pv, *npv; 758 759 /* 760 * Remove from the PV table. 761 */ 762 pv = pa_to_pv(pa); 763 if (!pv) 764 return; 765 766 /* 767 * If it is the first entry on the list, it is actually 768 * in the header and we must copy the following entry up 769 * to the header. Otherwise we must search the list for 770 * the entry. In either case we free the now unused entry. 771 */ 772 if (pm == pv->pv_pm && PV_CMPVA(va, pv)) { 773 if (PV_ISWIRED(pv)) { 774 pm->pm_stats.wired_count--; 775 } 776 if ((npv = pv->pv_next)) { 777 *pv = *npv; 778 pool_put(&pv_pool, npv); 779 } else 780 pv->pv_pm = NULL; 781 } else { 782 for (; (npv = pv->pv_next) != NULL; pv = npv) 783 if (pm == npv->pv_pm && PV_CMPVA(va, npv)) 784 break; 785 if (npv) { 786 pv->pv_next = npv->pv_next; 787 if (PV_ISWIRED(npv)) { 788 pm->pm_stats.wired_count--; 789 } 790 pool_put(&pv_pool, npv); 791 } 792 } 793 } 794 795 /* 796 * Insert physical page at pa into the given pmap at virtual address va. 797 */ 798 int 799 pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 800 { 801 int s; 802 u_int tte; 803 bool managed; 804 805 /* 806 * Have to remove any existing mapping first. 807 */ 808 pmap_remove(pm, va, va + PAGE_SIZE); 809 810 if (flags & PMAP_WIRED) 811 flags |= prot; 812 813 managed = uvm_pageismanaged(pa); 814 815 /* 816 * Generate TTE. 817 */ 818 tte = TTE_PA(pa); 819 /* XXXX -- need to support multiple page sizes. */ 820 tte |= TTE_SZ_16K; 821 #ifdef DIAGNOSTIC 822 if ((flags & (PME_NOCACHE | PME_WRITETHROUG)) == 823 (PME_NOCACHE | PME_WRITETHROUG)) 824 panic("pmap_enter: uncached & writethrough"); 825 #endif 826 if (flags & PME_NOCACHE) 827 /* Must be I/O mapping */ 828 tte |= TTE_I | TTE_G; 829 #ifdef PPC_4XX_NOCACHE 830 tte |= TTE_I; 831 #else 832 else if (flags & PME_WRITETHROUG) 833 /* Uncached and writethrough are not compatible */ 834 tte |= TTE_W; 835 #endif 836 if (pm == pmap_kernel()) 837 tte |= TTE_ZONE(ZONE_PRIV); 838 else 839 tte |= TTE_ZONE(ZONE_USER); 840 841 if (flags & VM_PROT_WRITE) 842 tte |= TTE_WR; 843 844 if (flags & VM_PROT_EXECUTE) 845 tte |= TTE_EX; 846 847 /* 848 * Now record mapping for later back-translation. 849 */ 850 if (pmap_initialized && managed) { 851 char *attr; 852 853 if (!pmap_enter_pv(pm, va, pa, flags)) { 854 /* Could not enter pv on a managed page */ 855 return 1; 856 } 857 858 /* Now set attributes. */ 859 attr = pa_to_attr(pa); 860 #ifdef DIAGNOSTIC 861 if (!attr) 862 panic("managed but no attr"); 863 #endif 864 if (flags & VM_PROT_ALL) 865 *attr |= PMAP_ATTR_REF; 866 if (flags & VM_PROT_WRITE) 867 *attr |= PMAP_ATTR_CHG; 868 } 869 870 s = splvm(); 871 872 /* Insert page into page table. */ 873 pte_enter(pm, va, tte); 874 875 /* If this is a real fault, enter it in the tlb */ 876 if (tte && ((flags & PMAP_WIRED) == 0)) { 877 ppc4xx_tlb_enter(pm->pm_ctx, va, tte); 878 } 879 splx(s); 880 881 /* Flush the real memory from the instruction cache. */ 882 if ((prot & VM_PROT_EXECUTE) && (tte & TTE_I) == 0) 883 __syncicache((void *)pa, PAGE_SIZE); 884 885 return 0; 886 } 887 888 void 889 pmap_unwire(struct pmap *pm, vaddr_t va) 890 { 891 struct pv_entry *pv; 892 paddr_t pa; 893 int s; 894 895 if (!pmap_extract(pm, va, &pa)) { 896 return; 897 } 898 899 pv = pa_to_pv(pa); 900 if (!pv) 901 return; 902 903 s = splvm(); 904 while (pv != NULL) { 905 if (pm == pv->pv_pm && PV_CMPVA(va, pv)) { 906 if (PV_ISWIRED(pv)) { 907 PV_UNWIRE(pv); 908 pm->pm_stats.wired_count--; 909 } 910 break; 911 } 912 pv = pv->pv_next; 913 } 914 splx(s); 915 } 916 917 void 918 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 919 { 920 int s; 921 u_int tte; 922 struct pmap *pm = pmap_kernel(); 923 924 /* 925 * Have to remove any existing mapping first. 926 */ 927 928 /* 929 * Generate TTE. 930 * 931 * XXXX 932 * 933 * Since the kernel does not handle execution privileges properly, 934 * we will handle read and execute permissions together. 935 */ 936 tte = 0; 937 if (prot & VM_PROT_ALL) { 938 939 tte = TTE_PA(pa) | TTE_EX | TTE_ZONE(ZONE_PRIV); 940 /* XXXX -- need to support multiple page sizes. */ 941 tte |= TTE_SZ_16K; 942 #ifdef DIAGNOSTIC 943 if ((prot & (PME_NOCACHE | PME_WRITETHROUG)) == 944 (PME_NOCACHE | PME_WRITETHROUG)) 945 panic("pmap_kenter_pa: uncached & writethrough"); 946 #endif 947 if (prot & PME_NOCACHE) 948 /* Must be I/O mapping */ 949 tte |= TTE_I | TTE_G; 950 #ifdef PPC_4XX_NOCACHE 951 tte |= TTE_I; 952 #else 953 else if (prot & PME_WRITETHROUG) 954 /* Uncached and writethrough are not compatible */ 955 tte |= TTE_W; 956 #endif 957 if (prot & VM_PROT_WRITE) 958 tte |= TTE_WR; 959 } 960 961 s = splvm(); 962 963 /* Insert page into page table. */ 964 pte_enter(pm, va, tte); 965 splx(s); 966 } 967 968 void 969 pmap_kremove(vaddr_t va, vsize_t len) 970 { 971 972 while (len > 0) { 973 pte_enter(pmap_kernel(), va, 0); 974 va += PAGE_SIZE; 975 len -= PAGE_SIZE; 976 } 977 } 978 979 /* 980 * Remove the given range of mapping entries. 981 */ 982 void 983 pmap_remove(struct pmap *pm, vaddr_t va, vaddr_t endva) 984 { 985 int s; 986 paddr_t pa; 987 volatile u_int *ptp; 988 989 s = splvm(); 990 while (va < endva) { 991 992 if ((ptp = pte_find(pm, va)) && (pa = *ptp)) { 993 pa = TTE_PA(pa); 994 pmap_remove_pv(pm, va, pa); 995 *ptp = 0; 996 ppc4xx_tlb_flush(va, pm->pm_ctx); 997 pm->pm_stats.resident_count--; 998 } 999 va += PAGE_SIZE; 1000 } 1001 1002 splx(s); 1003 } 1004 1005 /* 1006 * Get the physical page address for the given pmap/virtual address. 1007 */ 1008 bool 1009 pmap_extract(struct pmap *pm, vaddr_t va, paddr_t *pap) 1010 { 1011 int seg = STIDX(va); 1012 int ptn = PTIDX(va); 1013 u_int pa = 0; 1014 int s; 1015 1016 s = splvm(); 1017 if (pm->pm_ptbl[seg] && (pa = pm->pm_ptbl[seg][ptn])) { 1018 *pap = TTE_PA(pa) | (va & PGOFSET); 1019 } 1020 splx(s); 1021 return (pa != 0); 1022 } 1023 1024 /* 1025 * Lower the protection on the specified range of this pmap. 1026 * 1027 * There are only two cases: either the protection is going to 0, 1028 * or it is going to read-only. 1029 */ 1030 void 1031 pmap_protect(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1032 { 1033 volatile u_int *ptp; 1034 int s, bic; 1035 1036 if ((prot & VM_PROT_READ) == 0) { 1037 pmap_remove(pm, sva, eva); 1038 return; 1039 } 1040 bic = 0; 1041 if ((prot & VM_PROT_WRITE) == 0) { 1042 bic |= TTE_WR; 1043 } 1044 if ((prot & VM_PROT_EXECUTE) == 0) { 1045 bic |= TTE_EX; 1046 } 1047 if (bic == 0) { 1048 return; 1049 } 1050 s = splvm(); 1051 while (sva < eva) { 1052 if ((ptp = pte_find(pm, sva)) != NULL) { 1053 *ptp &= ~bic; 1054 ppc4xx_tlb_flush(sva, pm->pm_ctx); 1055 } 1056 sva += PAGE_SIZE; 1057 } 1058 splx(s); 1059 } 1060 1061 bool 1062 pmap_check_attr(struct vm_page *pg, u_int mask, int clear) 1063 { 1064 paddr_t pa; 1065 char *attr; 1066 int s, rv; 1067 1068 /* 1069 * First modify bits in cache. 1070 */ 1071 pa = VM_PAGE_TO_PHYS(pg); 1072 attr = pa_to_attr(pa); 1073 if (attr == NULL) 1074 return false; 1075 1076 s = splvm(); 1077 rv = ((*attr & mask) != 0); 1078 if (clear) { 1079 *attr &= ~mask; 1080 pmap_page_protect(pg, mask == PMAP_ATTR_CHG ? VM_PROT_READ : 0); 1081 } 1082 splx(s); 1083 return rv; 1084 } 1085 1086 1087 /* 1088 * Lower the protection on the specified physical page. 1089 * 1090 * There are only two cases: either the protection is going to 0, 1091 * or it is going to read-only. 1092 */ 1093 void 1094 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 1095 { 1096 paddr_t pa = VM_PAGE_TO_PHYS(pg); 1097 vaddr_t va; 1098 struct pv_entry *pvh, *pv, *npv; 1099 struct pmap *pm; 1100 1101 pvh = pa_to_pv(pa); 1102 if (pvh == NULL) 1103 return; 1104 1105 /* Handle extra pvs which may be deleted in the operation */ 1106 for (pv = pvh->pv_next; pv; pv = npv) { 1107 npv = pv->pv_next; 1108 1109 pm = pv->pv_pm; 1110 va = pv->pv_va; 1111 pmap_protect(pm, va, va + PAGE_SIZE, prot); 1112 } 1113 /* Now check the head pv */ 1114 if (pvh->pv_pm) { 1115 pv = pvh; 1116 pm = pv->pv_pm; 1117 va = pv->pv_va; 1118 pmap_protect(pm, va, va + PAGE_SIZE, prot); 1119 } 1120 } 1121 1122 /* 1123 * Activate the address space for the specified process. If the process 1124 * is the current process, load the new MMU context. 1125 */ 1126 void 1127 pmap_activate(struct lwp *l) 1128 { 1129 #if 0 1130 struct pcb *pcb = lwp_getpcb(l); 1131 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 1132 1133 /* 1134 * XXX Normally performed in cpu_lwp_fork(). 1135 */ 1136 printf("pmap_activate(%p), pmap=%p\n",l,pmap); 1137 pcb->pcb_pm = pmap; 1138 #endif 1139 } 1140 1141 /* 1142 * Deactivate the specified process's address space. 1143 */ 1144 void 1145 pmap_deactivate(struct lwp *l) 1146 { 1147 } 1148 1149 /* 1150 * Synchronize caches corresponding to [addr, addr+len) in p. 1151 */ 1152 void 1153 pmap_procwr(struct proc *p, vaddr_t va, size_t len) 1154 { 1155 struct pmap *pm = p->p_vmspace->vm_map.pmap; 1156 int msr, ctx, opid, step; 1157 1158 step = CACHELINESIZE; 1159 1160 /* 1161 * Need to turn off IMMU and switch to user context. 1162 * (icbi uses DMMU). 1163 */ 1164 if (!(ctx = pm->pm_ctx)) { 1165 /* No context -- assign it one */ 1166 ctx_alloc(pm); 1167 ctx = pm->pm_ctx; 1168 } 1169 __asm volatile("mfmsr %0;" 1170 "li %1, %7;" 1171 "andc %1,%0,%1;" 1172 "mtmsr %1;" 1173 "sync;isync;" 1174 "mfpid %1;" 1175 "mtpid %2;" 1176 "sync; isync;" 1177 "1:" 1178 "dcbf 0,%3;" 1179 "icbi 0,%3;" 1180 "add %3,%3,%5;" 1181 "addc. %4,%4,%6;" 1182 "bge 1b;" 1183 "mtpid %1;" 1184 "mtmsr %0;" 1185 "sync; isync" 1186 : "=&r" (msr), "=&r" (opid) 1187 : "r" (ctx), "r" (va), "r" (len), "r" (step), "r" (-step), 1188 "K" (PSL_IR | PSL_DR)); 1189 } 1190 1191 1192 /* This has to be done in real mode !!! */ 1193 void 1194 ppc4xx_tlb_flush(vaddr_t va, int pid) 1195 { 1196 u_long i, found; 1197 u_long msr; 1198 1199 /* If there's no context then it can't be mapped. */ 1200 if (!pid) 1201 return; 1202 1203 __asm( "mfpid %1;" /* Save PID */ 1204 "mfmsr %2;" /* Save MSR */ 1205 "li %0,0;" /* Now clear MSR */ 1206 "mtmsr %0;" 1207 "mtpid %4;" /* Set PID */ 1208 "sync;" 1209 "tlbsx. %0,0,%3;" /* Search TLB */ 1210 "sync;" 1211 "mtpid %1;" /* Restore PID */ 1212 "mtmsr %2;" /* Restore MSR */ 1213 "sync;isync;" 1214 "li %1,1;" 1215 "beq 1f;" 1216 "li %1,0;" 1217 "1:" 1218 : "=&r" (i), "=&r" (found), "=&r" (msr) 1219 : "r" (va), "r" (pid)); 1220 if (found && !TLB_LOCKED(i)) { 1221 1222 /* Now flush translation */ 1223 __asm volatile( 1224 "tlbwe %0,%1,0;" 1225 "sync;isync;" 1226 : : "r" (0), "r" (i)); 1227 1228 tlb_info[i].ti_ctx = 0; 1229 tlb_info[i].ti_flags = 0; 1230 tlbnext = i; 1231 /* Successful flushes */ 1232 tlbflush_ev.ev_count++; 1233 } 1234 } 1235 1236 void 1237 ppc4xx_tlb_flush_all(void) 1238 { 1239 u_long i; 1240 1241 for (i = 0; i < NTLB; i++) 1242 if (!TLB_LOCKED(i)) { 1243 __asm volatile( 1244 "tlbwe %0,%1,0;" 1245 "sync;isync;" 1246 : : "r" (0), "r" (i)); 1247 tlb_info[i].ti_ctx = 0; 1248 tlb_info[i].ti_flags = 0; 1249 } 1250 1251 __asm volatile("sync;isync"); 1252 } 1253 1254 /* Find a TLB entry to evict. */ 1255 static int 1256 ppc4xx_tlb_find_victim(void) 1257 { 1258 int flags; 1259 1260 for (;;) { 1261 if (++tlbnext >= NTLB) 1262 tlbnext = tlb_nreserved; 1263 flags = tlb_info[tlbnext].ti_flags; 1264 if (!(flags & TLBF_USED) || 1265 (flags & (TLBF_LOCKED | TLBF_REF)) == 0) { 1266 u_long va, stack = (u_long)&va; 1267 1268 if (!((tlb_info[tlbnext].ti_va ^ stack) & (~PGOFSET)) && 1269 (tlb_info[tlbnext].ti_ctx == KERNEL_PID) && 1270 (flags & TLBF_USED)) { 1271 /* Kernel stack page */ 1272 flags |= TLBF_USED; 1273 tlb_info[tlbnext].ti_flags = flags; 1274 } else { 1275 /* Found it! */ 1276 return (tlbnext); 1277 } 1278 } else { 1279 tlb_info[tlbnext].ti_flags = (flags & ~TLBF_REF); 1280 } 1281 } 1282 } 1283 1284 void 1285 ppc4xx_tlb_enter(int ctx, vaddr_t va, u_int pte) 1286 { 1287 u_long th, tl, idx; 1288 tlbpid_t pid; 1289 u_short msr; 1290 paddr_t pa; 1291 int s, sz; 1292 1293 tlbenter_ev.ev_count++; 1294 1295 sz = (pte & TTE_SZ_MASK) >> TTE_SZ_SHIFT; 1296 pa = (pte & TTE_RPN_MASK(sz)); 1297 th = (va & TLB_EPN_MASK) | (sz << TLB_SIZE_SHFT) | TLB_VALID; 1298 tl = (pte & ~TLB_RPN_MASK) | pa; 1299 tl |= ppc4xx_tlbflags(va, pa); 1300 1301 s = splhigh(); 1302 idx = ppc4xx_tlb_find_victim(); 1303 1304 #ifdef DIAGNOSTIC 1305 if ((idx < tlb_nreserved) || (idx >= NTLB)) { 1306 panic("ppc4xx_tlb_enter: replacing entry %ld", idx); 1307 } 1308 #endif 1309 1310 tlb_info[idx].ti_va = (va & TLB_EPN_MASK); 1311 tlb_info[idx].ti_ctx = ctx; 1312 tlb_info[idx].ti_flags = TLBF_USED | TLBF_REF; 1313 1314 __asm volatile( 1315 "mfmsr %0;" /* Save MSR */ 1316 "li %1,0;" 1317 "tlbwe %1,%3,0;" /* Invalidate old entry. */ 1318 "mtmsr %1;" /* Clear MSR */ 1319 "mfpid %1;" /* Save old PID */ 1320 "mtpid %2;" /* Load translation ctx */ 1321 "sync; isync;" 1322 #ifdef DEBUG 1323 "andi. %3,%3,63;" 1324 "tweqi %3,0;" /* XXXXX DEBUG trap on index 0 */ 1325 #endif 1326 "tlbwe %4,%3,1; tlbwe %5,%3,0;" /* Set TLB */ 1327 "sync; isync;" 1328 "mtpid %1; mtmsr %0;" /* Restore PID and MSR */ 1329 "sync; isync;" 1330 : "=&r" (msr), "=&r" (pid) 1331 : "r" (ctx), "r" (idx), "r" (tl), "r" (th)); 1332 splx(s); 1333 } 1334 1335 void 1336 ppc4xx_tlb_init(void) 1337 { 1338 int i; 1339 1340 /* Mark reserved TLB entries */ 1341 for (i = 0; i < tlb_nreserved; i++) { 1342 tlb_info[i].ti_flags = TLBF_LOCKED | TLBF_USED; 1343 tlb_info[i].ti_ctx = KERNEL_PID; 1344 } 1345 1346 /* Setup security zones */ 1347 /* Z0 - accessible by kernel only if TLB entry permissions allow 1348 * Z1,Z2 - access is controlled by TLB entry permissions 1349 * Z3 - full access regardless of TLB entry permissions 1350 */ 1351 1352 __asm volatile( 1353 "mtspr %0,%1;" 1354 "sync;" 1355 :: "K"(SPR_ZPR), "r" (0x1b000000)); 1356 } 1357 1358 /* 1359 * ppc4xx_tlb_size_mask: 1360 * 1361 * Roundup size to supported page size, return TLBHI mask and real size. 1362 */ 1363 static int 1364 ppc4xx_tlb_size_mask(size_t size, int *mask, int *rsiz) 1365 { 1366 int i; 1367 1368 for (i = 0; i < __arraycount(tlbsize); i++) 1369 if (size <= tlbsize[i]) { 1370 *mask = (i << TLB_SIZE_SHFT); 1371 *rsiz = tlbsize[i]; 1372 return (0); 1373 } 1374 return (EINVAL); 1375 } 1376 1377 /* 1378 * ppc4xx_tlb_mapiodev: 1379 * 1380 * Lookup virtual address of mapping previously entered via 1381 * ppc4xx_tlb_reserve. Search TLB directly so that we don't 1382 * need to waste extra storage for reserved mappings. Note 1383 * that reading TLBHI also sets PID, but all reserved mappings 1384 * use KERNEL_PID, so the side effect is nil. 1385 */ 1386 void * 1387 ppc4xx_tlb_mapiodev(paddr_t base, psize_t len) 1388 { 1389 paddr_t pa; 1390 vaddr_t va; 1391 u_int lo, hi, sz; 1392 int i; 1393 1394 /* tlb_nreserved is only allowed to grow, so this is safe. */ 1395 for (i = 0; i < tlb_nreserved; i++) { 1396 __asm volatile ( 1397 " tlbre %0,%2,1 \n" /* TLBLO */ 1398 " tlbre %1,%2,0 \n" /* TLBHI */ 1399 : "=&r" (lo), "=&r" (hi) 1400 : "r" (i)); 1401 1402 KASSERT(hi & TLB_VALID); 1403 KASSERT(mfspr(SPR_PID) == KERNEL_PID); 1404 1405 pa = (lo & TLB_RPN_MASK); 1406 if (base < pa) 1407 continue; 1408 1409 sz = tlbsize[(hi & TLB_SIZE_MASK) >> TLB_SIZE_SHFT]; 1410 if ((base + len) > (pa + sz)) 1411 continue; 1412 1413 va = (hi & TLB_EPN_MASK) + (base & (sz - 1)); /* sz = 2^n */ 1414 return (void *)(va); 1415 } 1416 1417 return (NULL); 1418 } 1419 1420 /* 1421 * ppc4xx_tlb_reserve: 1422 * 1423 * Map physical range to kernel virtual chunk via reserved TLB entry. 1424 */ 1425 void 1426 ppc4xx_tlb_reserve(paddr_t pa, vaddr_t va, size_t size, int flags) 1427 { 1428 u_int lo, hi; 1429 int szmask, rsize; 1430 1431 /* Called before pmap_bootstrap(), va outside kernel space. */ 1432 KASSERT(va < VM_MIN_KERNEL_ADDRESS || va >= VM_MAX_KERNEL_ADDRESS); 1433 KASSERT(! pmap_bootstrap_done); 1434 KASSERT(tlb_nreserved < NTLB); 1435 1436 /* Resolve size. */ 1437 if (ppc4xx_tlb_size_mask(size, &szmask, &rsize) != 0) 1438 panic("ppc4xx_tlb_reserve: entry %d, %zuB too large", 1439 size, tlb_nreserved); 1440 1441 /* Real size will be power of two >= 1024, so this is OK. */ 1442 pa &= ~(rsize - 1); /* RPN */ 1443 va &= ~(rsize - 1); /* EPN */ 1444 1445 lo = pa | TLB_WR | flags; 1446 hi = va | TLB_VALID | szmask; 1447 1448 #ifdef PPC_4XX_NOCACHE 1449 lo |= TLB_I; 1450 #endif 1451 1452 __asm volatile( 1453 " tlbwe %1,%0,1 \n" /* write TLBLO */ 1454 " tlbwe %2,%0,0 \n" /* write TLBHI */ 1455 " sync \n" 1456 " isync \n" 1457 : : "r" (tlb_nreserved), "r" (lo), "r" (hi)); 1458 1459 tlb_nreserved++; 1460 } 1461 1462 /* 1463 * We should pass the ctx in from trap code. 1464 */ 1465 int 1466 pmap_tlbmiss(vaddr_t va, int ctx) 1467 { 1468 volatile u_int *pte; 1469 u_long tte; 1470 1471 tlbmiss_ev.ev_count++; 1472 1473 /* 1474 * We will reserve 0 upto VM_MIN_KERNEL_ADDRESS for va == pa mappings. 1475 * Physical RAM is expected to live in this range, care must be taken 1476 * to not clobber 0 upto ${physmem} with device mappings in machdep 1477 * code. 1478 */ 1479 if (ctx != KERNEL_PID || 1480 (va >= VM_MIN_KERNEL_ADDRESS && va < VM_MAX_KERNEL_ADDRESS)) { 1481 pte = pte_find((struct pmap *)__UNVOLATILE(ctxbusy[ctx]), va); 1482 if (pte == NULL) { 1483 /* Map unmanaged addresses directly for kernel access */ 1484 return 1; 1485 } 1486 tte = *pte; 1487 if (tte == 0) { 1488 return 1; 1489 } 1490 } else { 1491 /* Create a 16MB writable mapping. */ 1492 #ifdef PPC_4XX_NOCACHE 1493 tte = TTE_PA(va) | TTE_ZONE(ZONE_PRIV) | TTE_SZ_16M | TTE_I |TTE_WR; 1494 #else 1495 tte = TTE_PA(va) | TTE_ZONE(ZONE_PRIV) | TTE_SZ_16M | TTE_WR; 1496 #endif 1497 } 1498 tlbhit_ev.ev_count++; 1499 ppc4xx_tlb_enter(ctx, va, tte); 1500 1501 return 0; 1502 } 1503 1504 /* 1505 * Flush all the entries matching a context from the TLB. 1506 */ 1507 static int 1508 ctx_flush(int cnum) 1509 { 1510 int i; 1511 1512 /* We gotta steal this context */ 1513 for (i = tlb_nreserved; i < NTLB; i++) { 1514 if (tlb_info[i].ti_ctx == cnum) { 1515 /* Can't steal ctx if it has a locked entry. */ 1516 if (TLB_LOCKED(i)) { 1517 #ifdef DIAGNOSTIC 1518 printf("ctx_flush: can't invalidate " 1519 "locked mapping %d " 1520 "for context %d\n", i, cnum); 1521 #ifdef DDB 1522 Debugger(); 1523 #endif 1524 #endif 1525 return (1); 1526 } 1527 #ifdef DIAGNOSTIC 1528 if (i < tlb_nreserved) 1529 panic("TLB entry %d not locked", i); 1530 #endif 1531 /* Invalidate particular TLB entry regardless of locked status */ 1532 __asm volatile("tlbwe %0,%1,0" : :"r"(0),"r"(i)); 1533 tlb_info[i].ti_flags = 0; 1534 } 1535 } 1536 return (0); 1537 } 1538 1539 /* 1540 * Allocate a context. If necessary, steal one from someone else. 1541 * 1542 * The new context is flushed from the TLB before returning. 1543 */ 1544 int 1545 ctx_alloc(struct pmap *pm) 1546 { 1547 int s, cnum; 1548 static int next = MINCTX; 1549 1550 if (pm == pmap_kernel()) { 1551 #ifdef DIAGNOSTIC 1552 printf("ctx_alloc: kernel pmap!\n"); 1553 #endif 1554 return (0); 1555 } 1556 s = splvm(); 1557 1558 /* Find a likely context. */ 1559 cnum = next; 1560 do { 1561 if ((++cnum) > NUMCTX) 1562 cnum = MINCTX; 1563 } while (ctxbusy[cnum] != NULL && cnum != next); 1564 1565 /* Now clean it out */ 1566 oops: 1567 if (cnum < MINCTX) 1568 cnum = MINCTX; /* Never steal ctx 0 or 1 */ 1569 if (ctx_flush(cnum)) { 1570 /* oops -- something's wired. */ 1571 if ((++cnum) > NUMCTX) 1572 cnum = MINCTX; 1573 goto oops; 1574 } 1575 1576 if (ctxbusy[cnum]) { 1577 #ifdef DEBUG 1578 /* We should identify this pmap and clear it */ 1579 printf("Warning: stealing context %d\n", cnum); 1580 #endif 1581 ctxbusy[cnum]->pm_ctx = 0; 1582 } 1583 ctxbusy[cnum] = pm; 1584 next = cnum; 1585 splx(s); 1586 pm->pm_ctx = cnum; 1587 1588 return cnum; 1589 } 1590 1591 /* 1592 * Give away a context. 1593 */ 1594 void 1595 ctx_free(struct pmap *pm) 1596 { 1597 int oldctx; 1598 1599 oldctx = pm->pm_ctx; 1600 1601 if (oldctx == 0) 1602 panic("ctx_free: freeing kernel context"); 1603 #ifdef DIAGNOSTIC 1604 if (ctxbusy[oldctx] == 0) 1605 printf("ctx_free: freeing free context %d\n", oldctx); 1606 if (ctxbusy[oldctx] != pm) { 1607 printf("ctx_free: freeing someone esle's context\n " 1608 "ctxbusy[%d] = %p, pm->pm_ctx = %p\n", 1609 oldctx, (void *)(u_long)ctxbusy[oldctx], pm); 1610 #ifdef DDB 1611 Debugger(); 1612 #endif 1613 } 1614 #endif 1615 /* We should verify it has not been stolen and reallocated... */ 1616 ctxbusy[oldctx] = NULL; 1617 ctx_flush(oldctx); 1618 } 1619 1620 1621 #ifdef DEBUG 1622 /* 1623 * Test ref/modify handling. 1624 */ 1625 void pmap_testout(void); 1626 void 1627 pmap_testout(void) 1628 { 1629 vaddr_t va; 1630 volatile int *loc; 1631 int val = 0; 1632 paddr_t pa; 1633 struct vm_page *pg; 1634 int ref, mod; 1635 1636 /* Allocate a page */ 1637 va = (vaddr_t)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 1638 UVM_KMF_WIRED | UVM_KMF_ZERO); 1639 loc = (int*)va; 1640 1641 pmap_extract(pmap_kernel(), va, &pa); 1642 pg = PHYS_TO_VM_PAGE(pa); 1643 pmap_unwire(pmap_kernel(), va); 1644 1645 pmap_kremove(va, PAGE_SIZE); 1646 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1647 pmap_update(pmap_kernel()); 1648 1649 /* Now clear reference and modify */ 1650 ref = pmap_clear_reference(pg); 1651 mod = pmap_clear_modify(pg); 1652 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1653 (void *)(u_long)va, (long)pa, 1654 ref, mod); 1655 1656 /* Check it's properly cleared */ 1657 ref = pmap_is_referenced(pg); 1658 mod = pmap_is_modified(pg); 1659 printf("Checking cleared page: ref %d, mod %d\n", 1660 ref, mod); 1661 1662 /* Reference page */ 1663 val = *loc; 1664 1665 ref = pmap_is_referenced(pg); 1666 mod = pmap_is_modified(pg); 1667 printf("Referenced page: ref %d, mod %d val %x\n", 1668 ref, mod, val); 1669 1670 /* Now clear reference and modify */ 1671 ref = pmap_clear_reference(pg); 1672 mod = pmap_clear_modify(pg); 1673 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1674 (void *)(u_long)va, (long)pa, 1675 ref, mod); 1676 1677 /* Modify page */ 1678 *loc = 1; 1679 1680 ref = pmap_is_referenced(pg); 1681 mod = pmap_is_modified(pg); 1682 printf("Modified page: ref %d, mod %d\n", 1683 ref, mod); 1684 1685 /* Now clear reference and modify */ 1686 ref = pmap_clear_reference(pg); 1687 mod = pmap_clear_modify(pg); 1688 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1689 (void *)(u_long)va, (long)pa, 1690 ref, mod); 1691 1692 /* Check it's properly cleared */ 1693 ref = pmap_is_referenced(pg); 1694 mod = pmap_is_modified(pg); 1695 printf("Checking cleared page: ref %d, mod %d\n", 1696 ref, mod); 1697 1698 /* Modify page */ 1699 *loc = 1; 1700 1701 ref = pmap_is_referenced(pg); 1702 mod = pmap_is_modified(pg); 1703 printf("Modified page: ref %d, mod %d\n", 1704 ref, mod); 1705 1706 /* Check pmap_protect() */ 1707 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_READ); 1708 pmap_update(pmap_kernel()); 1709 ref = pmap_is_referenced(pg); 1710 mod = pmap_is_modified(pg); 1711 printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n", 1712 ref, mod); 1713 1714 /* Now clear reference and modify */ 1715 ref = pmap_clear_reference(pg); 1716 mod = pmap_clear_modify(pg); 1717 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1718 (void *)(u_long)va, (long)pa, 1719 ref, mod); 1720 1721 /* Reference page */ 1722 val = *loc; 1723 1724 ref = pmap_is_referenced(pg); 1725 mod = pmap_is_modified(pg); 1726 printf("Referenced page: ref %d, mod %d val %x\n", 1727 ref, mod, val); 1728 1729 /* Now clear reference and modify */ 1730 ref = pmap_clear_reference(pg); 1731 mod = pmap_clear_modify(pg); 1732 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1733 (void *)(u_long)va, (long)pa, 1734 ref, mod); 1735 1736 /* Modify page */ 1737 #if 0 1738 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1739 pmap_update(pmap_kernel()); 1740 #endif 1741 *loc = 1; 1742 1743 ref = pmap_is_referenced(pg); 1744 mod = pmap_is_modified(pg); 1745 printf("Modified page: ref %d, mod %d\n", 1746 ref, mod); 1747 1748 /* Check pmap_protect() */ 1749 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_NONE); 1750 pmap_update(pmap_kernel()); 1751 ref = pmap_is_referenced(pg); 1752 mod = pmap_is_modified(pg); 1753 printf("pmap_protect(): ref %d, mod %d\n", 1754 ref, mod); 1755 1756 /* Now clear reference and modify */ 1757 ref = pmap_clear_reference(pg); 1758 mod = pmap_clear_modify(pg); 1759 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1760 (void *)(u_long)va, (long)pa, 1761 ref, mod); 1762 1763 /* Reference page */ 1764 val = *loc; 1765 1766 ref = pmap_is_referenced(pg); 1767 mod = pmap_is_modified(pg); 1768 printf("Referenced page: ref %d, mod %d val %x\n", 1769 ref, mod, val); 1770 1771 /* Now clear reference and modify */ 1772 ref = pmap_clear_reference(pg); 1773 mod = pmap_clear_modify(pg); 1774 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1775 (void *)(u_long)va, (long)pa, 1776 ref, mod); 1777 1778 /* Modify page */ 1779 #if 0 1780 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1781 pmap_update(pmap_kernel()); 1782 #endif 1783 *loc = 1; 1784 1785 ref = pmap_is_referenced(pg); 1786 mod = pmap_is_modified(pg); 1787 printf("Modified page: ref %d, mod %d\n", 1788 ref, mod); 1789 1790 /* Check pmap_pag_protect() */ 1791 pmap_page_protect(pg, VM_PROT_READ); 1792 ref = pmap_is_referenced(pg); 1793 mod = pmap_is_modified(pg); 1794 printf("pmap_page_protect(VM_PROT_READ): ref %d, mod %d\n", 1795 ref, mod); 1796 1797 /* Now clear reference and modify */ 1798 ref = pmap_clear_reference(pg); 1799 mod = pmap_clear_modify(pg); 1800 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1801 (void *)(u_long)va, (long)pa, 1802 ref, mod); 1803 1804 /* Reference page */ 1805 val = *loc; 1806 1807 ref = pmap_is_referenced(pg); 1808 mod = pmap_is_modified(pg); 1809 printf("Referenced page: ref %d, mod %d val %x\n", 1810 ref, mod, val); 1811 1812 /* Now clear reference and modify */ 1813 ref = pmap_clear_reference(pg); 1814 mod = pmap_clear_modify(pg); 1815 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1816 (void *)(u_long)va, (long)pa, 1817 ref, mod); 1818 1819 /* Modify page */ 1820 #if 0 1821 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1822 pmap_update(pmap_kernel()); 1823 #endif 1824 *loc = 1; 1825 1826 ref = pmap_is_referenced(pg); 1827 mod = pmap_is_modified(pg); 1828 printf("Modified page: ref %d, mod %d\n", 1829 ref, mod); 1830 1831 /* Check pmap_pag_protect() */ 1832 pmap_page_protect(pg, VM_PROT_NONE); 1833 ref = pmap_is_referenced(pg); 1834 mod = pmap_is_modified(pg); 1835 printf("pmap_page_protect(): ref %d, mod %d\n", 1836 ref, mod); 1837 1838 /* Now clear reference and modify */ 1839 ref = pmap_clear_reference(pg); 1840 mod = pmap_clear_modify(pg); 1841 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1842 (void *)(u_long)va, (long)pa, 1843 ref, mod); 1844 1845 1846 /* Reference page */ 1847 val = *loc; 1848 1849 ref = pmap_is_referenced(pg); 1850 mod = pmap_is_modified(pg); 1851 printf("Referenced page: ref %d, mod %d val %x\n", 1852 ref, mod, val); 1853 1854 /* Now clear reference and modify */ 1855 ref = pmap_clear_reference(pg); 1856 mod = pmap_clear_modify(pg); 1857 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1858 (void *)(u_long)va, (long)pa, 1859 ref, mod); 1860 1861 /* Modify page */ 1862 #if 0 1863 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1864 pmap_update(pmap_kernel()); 1865 #endif 1866 *loc = 1; 1867 1868 ref = pmap_is_referenced(pg); 1869 mod = pmap_is_modified(pg); 1870 printf("Modified page: ref %d, mod %d\n", 1871 ref, mod); 1872 1873 /* Unmap page */ 1874 pmap_remove(pmap_kernel(), va, va+1); 1875 pmap_update(pmap_kernel()); 1876 ref = pmap_is_referenced(pg); 1877 mod = pmap_is_modified(pg); 1878 printf("Unmapped page: ref %d, mod %d\n", ref, mod); 1879 1880 /* Now clear reference and modify */ 1881 ref = pmap_clear_reference(pg); 1882 mod = pmap_clear_modify(pg); 1883 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1884 (void *)(u_long)va, (long)pa, ref, mod); 1885 1886 /* Check it's properly cleared */ 1887 ref = pmap_is_referenced(pg); 1888 mod = pmap_is_modified(pg); 1889 printf("Checking cleared page: ref %d, mod %d\n", 1890 ref, mod); 1891 1892 pmap_remove(pmap_kernel(), va, va + PAGE_SIZE); 1893 pmap_kenter_pa(va, pa, VM_PROT_ALL, 0); 1894 uvm_km_free(kernel_map, (vaddr_t)va, PAGE_SIZE, UVM_KMF_WIRED); 1895 } 1896 #endif 1897