1 /* $NetBSD: pmap.c,v 1.43 2006/10/16 18:14:38 kiyohara Exp $ */ 2 3 /* 4 * Copyright 2001 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Eduardo Horvath and Simon Burge for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 40 * Copyright (C) 1995, 1996 TooLs GmbH. 41 * All rights reserved. 42 * 43 * Redistribution and use in source and binary forms, with or without 44 * modification, are permitted provided that the following conditions 45 * are met: 46 * 1. Redistributions of source code must retain the above copyright 47 * notice, this list of conditions and the following disclaimer. 48 * 2. Redistributions in binary form must reproduce the above copyright 49 * notice, this list of conditions and the following disclaimer in the 50 * documentation and/or other materials provided with the distribution. 51 * 3. All advertising materials mentioning features or use of this software 52 * must display the following acknowledgement: 53 * This product includes software developed by TooLs GmbH. 54 * 4. The name of TooLs GmbH may not be used to endorse or promote products 55 * derived from this software without specific prior written permission. 56 * 57 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 58 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 59 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 60 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 61 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 62 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 63 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 64 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 65 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 66 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 67 */ 68 69 #include <sys/cdefs.h> 70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.43 2006/10/16 18:14:38 kiyohara Exp $"); 71 72 #include <sys/param.h> 73 #include <sys/malloc.h> 74 #include <sys/proc.h> 75 #include <sys/user.h> 76 #include <sys/queue.h> 77 #include <sys/systm.h> 78 #include <sys/pool.h> 79 #include <sys/device.h> 80 81 #include <uvm/uvm.h> 82 83 #include <machine/cpu.h> 84 #include <machine/pcb.h> 85 #include <machine/powerpc.h> 86 87 #include <powerpc/spr.h> 88 #include <machine/tlb.h> 89 90 /* 91 * kernmap is an array of PTEs large enough to map in 92 * 4GB. At 16KB/page it is 256K entries or 2MB. 93 */ 94 #define KERNMAP_SIZE ((0xffffffffU/PAGE_SIZE)+1) 95 caddr_t kernmap; 96 97 #define MINCTX 2 98 #define NUMCTX 256 99 100 volatile struct pmap *ctxbusy[NUMCTX]; 101 102 #define TLBF_USED 0x1 103 #define TLBF_REF 0x2 104 #define TLBF_LOCKED 0x4 105 #define TLB_LOCKED(i) (tlb_info[(i)].ti_flags & TLBF_LOCKED) 106 107 typedef struct tlb_info_s { 108 char ti_flags; 109 char ti_ctx; /* TLB_PID assiciated with the entry */ 110 u_int ti_va; 111 } tlb_info_t; 112 113 volatile tlb_info_t tlb_info[NTLB]; 114 /* We'll use a modified FIFO replacement policy cause it's cheap */ 115 volatile int tlbnext; 116 117 static int tlb_nreserved = 0; 118 static int pmap_bootstrap_done = 0; 119 120 /* Event counters */ 121 struct evcnt tlbmiss_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 122 NULL, "cpu", "tlbmiss"); 123 struct evcnt tlbhit_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 124 NULL, "cpu", "tlbhit"); 125 struct evcnt tlbflush_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 126 NULL, "cpu", "tlbflush"); 127 struct evcnt tlbenter_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 128 NULL, "cpu", "tlbenter"); 129 130 struct pmap kernel_pmap_; 131 132 int physmem; 133 static int npgs; 134 static u_int nextavail; 135 #ifndef MSGBUFADDR 136 extern paddr_t msgbuf_paddr; 137 #endif 138 139 static struct mem_region *mem, *avail; 140 141 /* 142 * This is a cache of referenced/modified bits. 143 * Bits herein are shifted by ATTRSHFT. 144 */ 145 static char *pmap_attrib; 146 147 #define PV_WIRED 0x1 148 #define PV_WIRE(pv) ((pv)->pv_va |= PV_WIRED) 149 #define PV_UNWIRE(pv) ((pv)->pv_va &= ~PV_WIRED) 150 #define PV_ISWIRED(pv) ((pv)->pv_va & PV_WIRED) 151 #define PV_CMPVA(va,pv) (!(((pv)->pv_va ^ (va)) & (~PV_WIRED))) 152 153 struct pv_entry { 154 struct pv_entry *pv_next; /* Linked list of mappings */ 155 vaddr_t pv_va; /* virtual address of mapping */ 156 struct pmap *pv_pm; 157 }; 158 159 /* Each index corresponds to TLB_SIZE_* value. */ 160 static size_t tlbsize[] = { 161 1024, /* TLB_SIZE_1K */ 162 4096, /* TLB_SIZE_4K */ 163 16384, /* TLB_SIZE_16K */ 164 65536, /* TLB_SIZE_64K */ 165 262144, /* TLB_SIZE_256K */ 166 1048576, /* TLB_SIZE_1M */ 167 4194304, /* TLB_SIZE_4M */ 168 16777216, /* TLB_SIZE_16M */ 169 }; 170 171 struct pv_entry *pv_table; 172 static struct pool pv_pool; 173 174 static int pmap_initialized; 175 176 static int ctx_flush(int); 177 178 inline struct pv_entry *pa_to_pv(paddr_t); 179 static inline char *pa_to_attr(paddr_t); 180 181 static inline volatile u_int *pte_find(struct pmap *, vaddr_t); 182 static inline int pte_enter(struct pmap *, vaddr_t, u_int); 183 184 static inline int pmap_enter_pv(struct pmap *, vaddr_t, paddr_t, boolean_t); 185 static void pmap_remove_pv(struct pmap *, vaddr_t, paddr_t); 186 187 static int ppc4xx_tlb_size_mask(size_t, int *, int *); 188 189 190 inline struct pv_entry * 191 pa_to_pv(paddr_t pa) 192 { 193 int bank, pg; 194 195 bank = vm_physseg_find(atop(pa), &pg); 196 if (bank == -1) 197 return NULL; 198 return &vm_physmem[bank].pmseg.pvent[pg]; 199 } 200 201 static inline char * 202 pa_to_attr(paddr_t pa) 203 { 204 int bank, pg; 205 206 bank = vm_physseg_find(atop(pa), &pg); 207 if (bank == -1) 208 return NULL; 209 return &vm_physmem[bank].pmseg.attrs[pg]; 210 } 211 212 /* 213 * Insert PTE into page table. 214 */ 215 int 216 pte_enter(struct pmap *pm, vaddr_t va, u_int pte) 217 { 218 int seg = STIDX(va); 219 int ptn = PTIDX(va); 220 u_int oldpte; 221 222 if (!pm->pm_ptbl[seg]) { 223 /* Don't allocate a page to clear a non-existent mapping. */ 224 if (!pte) 225 return (0); 226 /* Allocate a page XXXX this will sleep! */ 227 pm->pm_ptbl[seg] = 228 (uint *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 229 UVM_KMF_WIRED | UVM_KMF_ZERO); 230 } 231 oldpte = pm->pm_ptbl[seg][ptn]; 232 pm->pm_ptbl[seg][ptn] = pte; 233 234 /* Flush entry. */ 235 ppc4xx_tlb_flush(va, pm->pm_ctx); 236 if (oldpte != pte) { 237 if (pte == 0) 238 pm->pm_stats.resident_count--; 239 else 240 pm->pm_stats.resident_count++; 241 } 242 return (1); 243 } 244 245 /* 246 * Get a pointer to a PTE in a page table. 247 */ 248 volatile u_int * 249 pte_find(struct pmap *pm, vaddr_t va) 250 { 251 int seg = STIDX(va); 252 int ptn = PTIDX(va); 253 254 if (pm->pm_ptbl[seg]) 255 return (&pm->pm_ptbl[seg][ptn]); 256 257 return (NULL); 258 } 259 260 /* 261 * This is called during initppc, before the system is really initialized. 262 */ 263 void 264 pmap_bootstrap(u_int kernelstart, u_int kernelend) 265 { 266 struct mem_region *mp, *mp1; 267 int cnt, i; 268 u_int s, e, sz; 269 270 tlbnext = tlb_nreserved; 271 272 /* 273 * Allocate the kernel page table at the end of 274 * kernel space so it's in the locked TTE. 275 */ 276 kernmap = (caddr_t)kernelend; 277 278 /* 279 * Initialize kernel page table. 280 */ 281 for (i = 0; i < STSZ; i++) { 282 pmap_kernel()->pm_ptbl[i] = 0; 283 } 284 ctxbusy[0] = ctxbusy[1] = pmap_kernel(); 285 286 /* 287 * Announce page-size to the VM-system 288 */ 289 uvmexp.pagesize = NBPG; 290 uvm_setpagesize(); 291 292 /* 293 * Get memory. 294 */ 295 mem_regions(&mem, &avail); 296 for (mp = mem; mp->size; mp++) { 297 physmem += btoc(mp->size); 298 printf("+%lx,",mp->size); 299 } 300 printf("\n"); 301 ppc4xx_tlb_init(); 302 /* 303 * Count the number of available entries. 304 */ 305 for (cnt = 0, mp = avail; mp->size; mp++) 306 cnt++; 307 308 /* 309 * Page align all regions. 310 * Non-page aligned memory isn't very interesting to us. 311 * Also, sort the entries for ascending addresses. 312 */ 313 kernelstart &= ~PGOFSET; 314 kernelend = (kernelend + PGOFSET) & ~PGOFSET; 315 for (mp = avail; mp->size; mp++) { 316 s = mp->start; 317 e = mp->start + mp->size; 318 printf("%08x-%08x -> ",s,e); 319 /* 320 * Check whether this region holds all of the kernel. 321 */ 322 if (s < kernelstart && e > kernelend) { 323 avail[cnt].start = kernelend; 324 avail[cnt++].size = e - kernelend; 325 e = kernelstart; 326 } 327 /* 328 * Look whether this regions starts within the kernel. 329 */ 330 if (s >= kernelstart && s < kernelend) { 331 if (e <= kernelend) 332 goto empty; 333 s = kernelend; 334 } 335 /* 336 * Now look whether this region ends within the kernel. 337 */ 338 if (e > kernelstart && e <= kernelend) { 339 if (s >= kernelstart) 340 goto empty; 341 e = kernelstart; 342 } 343 /* 344 * Now page align the start and size of the region. 345 */ 346 s = round_page(s); 347 e = trunc_page(e); 348 if (e < s) 349 e = s; 350 sz = e - s; 351 printf("%08x-%08x = %x\n",s,e,sz); 352 /* 353 * Check whether some memory is left here. 354 */ 355 if (sz == 0) { 356 empty: 357 memmove(mp, mp + 1, 358 (cnt - (mp - avail)) * sizeof *mp); 359 cnt--; 360 mp--; 361 continue; 362 } 363 /* 364 * Do an insertion sort. 365 */ 366 npgs += btoc(sz); 367 for (mp1 = avail; mp1 < mp; mp1++) 368 if (s < mp1->start) 369 break; 370 if (mp1 < mp) { 371 memmove(mp1 + 1, mp1, (char *)mp - (char *)mp1); 372 mp1->start = s; 373 mp1->size = sz; 374 } else { 375 mp->start = s; 376 mp->size = sz; 377 } 378 } 379 380 /* 381 * We cannot do pmap_steal_memory here, 382 * since we don't run with translation enabled yet. 383 */ 384 #ifndef MSGBUFADDR 385 /* 386 * allow for msgbuf 387 */ 388 sz = round_page(MSGBUFSIZE); 389 mp = NULL; 390 for (mp1 = avail; mp1->size; mp1++) 391 if (mp1->size >= sz) 392 mp = mp1; 393 if (mp == NULL) 394 panic("not enough memory?"); 395 396 npgs -= btoc(sz); 397 msgbuf_paddr = mp->start + mp->size - sz; 398 mp->size -= sz; 399 if (mp->size <= 0) 400 memmove(mp, mp + 1, (cnt - (mp - avail)) * sizeof *mp); 401 #endif 402 403 for (mp = avail; mp->size; mp++) 404 uvm_page_physload(atop(mp->start), atop(mp->start + mp->size), 405 atop(mp->start), atop(mp->start + mp->size), 406 VM_FREELIST_DEFAULT); 407 408 /* 409 * Initialize kernel pmap and hardware. 410 */ 411 /* Setup TLB pid allocator so it knows we alreadu using PID 1 */ 412 pmap_kernel()->pm_ctx = KERNEL_PID; 413 nextavail = avail->start; 414 415 evcnt_attach_static(&tlbmiss_ev); 416 evcnt_attach_static(&tlbhit_ev); 417 evcnt_attach_static(&tlbflush_ev); 418 evcnt_attach_static(&tlbenter_ev); 419 420 pmap_bootstrap_done = 1; 421 } 422 423 /* 424 * Restrict given range to physical memory 425 * 426 * (Used by /dev/mem) 427 */ 428 void 429 pmap_real_memory(paddr_t *start, psize_t *size) 430 { 431 struct mem_region *mp; 432 433 for (mp = mem; mp->size; mp++) { 434 if (*start + *size > mp->start && 435 *start < mp->start + mp->size) { 436 if (*start < mp->start) { 437 *size -= mp->start - *start; 438 *start = mp->start; 439 } 440 if (*start + *size > mp->start + mp->size) 441 *size = mp->start + mp->size - *start; 442 return; 443 } 444 } 445 *size = 0; 446 } 447 448 /* 449 * Initialize anything else for pmap handling. 450 * Called during vm_init(). 451 */ 452 void 453 pmap_init(void) 454 { 455 struct pv_entry *pv; 456 vsize_t sz; 457 vaddr_t addr; 458 int i, s; 459 int bank; 460 char *attr; 461 462 sz = (vsize_t)((sizeof(struct pv_entry) + 1) * npgs); 463 sz = round_page(sz); 464 addr = uvm_km_alloc(kernel_map, sz, 0, UVM_KMF_WIRED | UVM_KMF_ZERO); 465 s = splvm(); 466 pv = pv_table = (struct pv_entry *)addr; 467 for (i = npgs; --i >= 0;) 468 pv++->pv_pm = NULL; 469 pmap_attrib = (char *)pv; 470 memset(pv, 0, npgs); 471 472 pv = pv_table; 473 attr = pmap_attrib; 474 for (bank = 0; bank < vm_nphysseg; bank++) { 475 sz = vm_physmem[bank].end - vm_physmem[bank].start; 476 vm_physmem[bank].pmseg.pvent = pv; 477 vm_physmem[bank].pmseg.attrs = attr; 478 pv += sz; 479 attr += sz; 480 } 481 482 pmap_initialized = 1; 483 splx(s); 484 485 /* Setup a pool for additional pvlist structures */ 486 pool_init(&pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pv_entry", NULL); 487 } 488 489 /* 490 * How much virtual space is available to the kernel? 491 */ 492 void 493 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 494 { 495 496 #if 0 497 /* 498 * Reserve one segment for kernel virtual memory 499 */ 500 *start = (vaddr_t)(KERNEL_SR << ADDR_SR_SHFT); 501 *end = *start + SEGMENT_LENGTH; 502 #else 503 *start = (vaddr_t) VM_MIN_KERNEL_ADDRESS; 504 *end = (vaddr_t) VM_MAX_KERNEL_ADDRESS; 505 #endif 506 } 507 508 #ifdef PMAP_GROWKERNEL 509 /* 510 * Preallocate kernel page tables to a specified VA. 511 * This simply loops through the first TTE for each 512 * page table from the beginning of the kernel pmap, 513 * reads the entry, and if the result is 514 * zero (either invalid entry or no page table) it stores 515 * a zero there, populating page tables in the process. 516 * This is not the most efficient technique but i don't 517 * expect it to be called that often. 518 */ 519 extern struct vm_page *vm_page_alloc1 __P((void)); 520 extern void vm_page_free1 __P((struct vm_page *)); 521 522 vaddr_t kbreak = VM_MIN_KERNEL_ADDRESS; 523 524 vaddr_t 525 pmap_growkernel(vaddr_t maxkvaddr) 526 { 527 int s; 528 int seg; 529 paddr_t pg; 530 struct pmap *pm = pmap_kernel(); 531 532 s = splvm(); 533 534 /* Align with the start of a page table */ 535 for (kbreak &= ~(PTMAP-1); kbreak < maxkvaddr; 536 kbreak += PTMAP) { 537 seg = STIDX(kbreak); 538 539 if (pte_find(pm, kbreak)) 540 continue; 541 542 if (uvm.page_init_done) { 543 pg = (paddr_t)VM_PAGE_TO_PHYS(vm_page_alloc1()); 544 } else { 545 if (!uvm_page_physget(&pg)) 546 panic("pmap_growkernel: no memory"); 547 } 548 if (!pg) 549 panic("pmap_growkernel: no pages"); 550 pmap_zero_page((paddr_t)pg); 551 552 /* XXX This is based on all phymem being addressable */ 553 pm->pm_ptbl[seg] = (u_int *)pg; 554 } 555 splx(s); 556 return (kbreak); 557 } 558 559 /* 560 * vm_page_alloc1: 561 * 562 * Allocate and return a memory cell with no associated object. 563 */ 564 struct vm_page * 565 vm_page_alloc1(void) 566 { 567 struct vm_page *pg; 568 569 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE); 570 if (pg) { 571 pg->wire_count = 1; /* no mappings yet */ 572 pg->flags &= ~PG_BUSY; /* never busy */ 573 } 574 return pg; 575 } 576 577 /* 578 * vm_page_free1: 579 * 580 * Returns the given page to the free list, 581 * disassociating it with any VM object. 582 * 583 * Object and page must be locked prior to entry. 584 */ 585 void 586 vm_page_free1(struct vm_page *pg) 587 { 588 #ifdef DIAGNOSTIC 589 if (pg->flags != (PG_CLEAN|PG_FAKE)) { 590 printf("Freeing invalid page %p\n", pg); 591 printf("pa = %llx\n", (unsigned long long)VM_PAGE_TO_PHYS(pg)); 592 #ifdef DDB 593 Debugger(); 594 #endif 595 return; 596 } 597 #endif 598 pg->flags |= PG_BUSY; 599 pg->wire_count = 0; 600 uvm_pagefree(pg); 601 } 602 #endif 603 604 /* 605 * Create and return a physical map. 606 */ 607 struct pmap * 608 pmap_create(void) 609 { 610 struct pmap *pm; 611 612 pm = malloc(sizeof *pm, M_VMPMAP, M_WAITOK); 613 memset(pm, 0, sizeof *pm); 614 pm->pm_refs = 1; 615 return pm; 616 } 617 618 /* 619 * Add a reference to the given pmap. 620 */ 621 void 622 pmap_reference(struct pmap *pm) 623 { 624 625 pm->pm_refs++; 626 } 627 628 /* 629 * Retire the given pmap from service. 630 * Should only be called if the map contains no valid mappings. 631 */ 632 void 633 pmap_destroy(struct pmap *pm) 634 { 635 int i; 636 637 if (--pm->pm_refs > 0) { 638 return; 639 } 640 KASSERT(pm->pm_stats.resident_count == 0); 641 KASSERT(pm->pm_stats.wired_count == 0); 642 for (i = 0; i < STSZ; i++) 643 if (pm->pm_ptbl[i]) { 644 uvm_km_free(kernel_map, (vaddr_t)pm->pm_ptbl[i], 645 PAGE_SIZE, UVM_KMF_WIRED); 646 pm->pm_ptbl[i] = NULL; 647 } 648 if (pm->pm_ctx) 649 ctx_free(pm); 650 free(pm, M_VMPMAP); 651 } 652 653 /* 654 * Copy the range specified by src_addr/len 655 * from the source map to the range dst_addr/len 656 * in the destination map. 657 * 658 * This routine is only advisory and need not do anything. 659 */ 660 void 661 pmap_copy(struct pmap *dst_pmap, struct pmap *src_pmap, vaddr_t dst_addr, 662 vsize_t len, vaddr_t src_addr) 663 { 664 } 665 666 /* 667 * Require that all active physical maps contain no 668 * incorrect entries NOW. 669 */ 670 void 671 pmap_update(struct pmap *pmap) 672 { 673 } 674 675 /* 676 * Garbage collects the physical map system for 677 * pages which are no longer used. 678 * Success need not be guaranteed -- that is, there 679 * may well be pages which are not referenced, but 680 * others may be collected. 681 * Called by the pageout daemon when pages are scarce. 682 */ 683 void 684 pmap_collect(struct pmap *pm) 685 { 686 } 687 688 /* 689 * Fill the given physical page with zeroes. 690 */ 691 void 692 pmap_zero_page(paddr_t pa) 693 { 694 695 #ifdef PPC_4XX_NOCACHE 696 memset((caddr_t)pa, 0, PAGE_SIZE); 697 #else 698 int i; 699 700 for (i = PAGE_SIZE/CACHELINESIZE; i > 0; i--) { 701 __asm volatile ("dcbz 0,%0" :: "r"(pa)); 702 pa += CACHELINESIZE; 703 } 704 #endif 705 } 706 707 /* 708 * Copy the given physical source page to its destination. 709 */ 710 void 711 pmap_copy_page(paddr_t src, paddr_t dst) 712 { 713 714 memcpy((caddr_t)dst, (caddr_t)src, PAGE_SIZE); 715 dcache_flush_page(dst); 716 } 717 718 /* 719 * This returns whether this is the first mapping of a page. 720 */ 721 static inline int 722 pmap_enter_pv(struct pmap *pm, vaddr_t va, paddr_t pa, boolean_t wired) 723 { 724 struct pv_entry *pv, *npv = NULL; 725 int s; 726 727 if (!pmap_initialized) 728 return 0; 729 730 s = splvm(); 731 pv = pa_to_pv(pa); 732 if (!pv->pv_pm) { 733 /* 734 * No entries yet, use header as the first entry. 735 */ 736 pv->pv_va = va; 737 pv->pv_pm = pm; 738 pv->pv_next = NULL; 739 } else { 740 /* 741 * There is at least one other VA mapping this page. 742 * Place this entry after the header. 743 */ 744 npv = pool_get(&pv_pool, PR_WAITOK); 745 npv->pv_va = va; 746 npv->pv_pm = pm; 747 npv->pv_next = pv->pv_next; 748 pv->pv_next = npv; 749 pv = npv; 750 } 751 if (wired) { 752 PV_WIRE(pv); 753 pm->pm_stats.wired_count++; 754 } 755 splx(s); 756 return (1); 757 } 758 759 static void 760 pmap_remove_pv(struct pmap *pm, vaddr_t va, paddr_t pa) 761 { 762 struct pv_entry *pv, *npv; 763 764 /* 765 * Remove from the PV table. 766 */ 767 pv = pa_to_pv(pa); 768 if (!pv) 769 return; 770 771 /* 772 * If it is the first entry on the list, it is actually 773 * in the header and we must copy the following entry up 774 * to the header. Otherwise we must search the list for 775 * the entry. In either case we free the now unused entry. 776 */ 777 if (pm == pv->pv_pm && PV_CMPVA(va, pv)) { 778 if (PV_ISWIRED(pv)) { 779 pm->pm_stats.wired_count--; 780 } 781 if ((npv = pv->pv_next)) { 782 *pv = *npv; 783 pool_put(&pv_pool, npv); 784 } else 785 pv->pv_pm = NULL; 786 } else { 787 for (; (npv = pv->pv_next) != NULL; pv = npv) 788 if (pm == npv->pv_pm && PV_CMPVA(va, npv)) 789 break; 790 if (npv) { 791 pv->pv_next = npv->pv_next; 792 if (PV_ISWIRED(npv)) { 793 pm->pm_stats.wired_count--; 794 } 795 pool_put(&pv_pool, npv); 796 } 797 } 798 } 799 800 /* 801 * Insert physical page at pa into the given pmap at virtual address va. 802 */ 803 int 804 pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags) 805 { 806 int s; 807 u_int tte; 808 int managed; 809 810 /* 811 * Have to remove any existing mapping first. 812 */ 813 pmap_remove(pm, va, va + PAGE_SIZE); 814 815 if (flags & PMAP_WIRED) 816 flags |= prot; 817 818 managed = 0; 819 if (vm_physseg_find(atop(pa), NULL) != -1) 820 managed = 1; 821 822 /* 823 * Generate TTE. 824 */ 825 tte = TTE_PA(pa); 826 /* XXXX -- need to support multiple page sizes. */ 827 tte |= TTE_SZ_16K; 828 #ifdef DIAGNOSTIC 829 if ((flags & (PME_NOCACHE | PME_WRITETHROUG)) == 830 (PME_NOCACHE | PME_WRITETHROUG)) 831 panic("pmap_enter: uncached & writethrough"); 832 #endif 833 if (flags & PME_NOCACHE) 834 /* Must be I/O mapping */ 835 tte |= TTE_I | TTE_G; 836 #ifdef PPC_4XX_NOCACHE 837 tte |= TTE_I; 838 #else 839 else if (flags & PME_WRITETHROUG) 840 /* Uncached and writethrough are not compatible */ 841 tte |= TTE_W; 842 #endif 843 if (pm == pmap_kernel()) 844 tte |= TTE_ZONE(ZONE_PRIV); 845 else 846 tte |= TTE_ZONE(ZONE_USER); 847 848 if (flags & VM_PROT_WRITE) 849 tte |= TTE_WR; 850 851 if (flags & VM_PROT_EXECUTE) 852 tte |= TTE_EX; 853 854 /* 855 * Now record mapping for later back-translation. 856 */ 857 if (pmap_initialized && managed) { 858 char *attr; 859 860 if (!pmap_enter_pv(pm, va, pa, flags & PMAP_WIRED)) { 861 /* Could not enter pv on a managed page */ 862 return 1; 863 } 864 865 /* Now set attributes. */ 866 attr = pa_to_attr(pa); 867 #ifdef DIAGNOSTIC 868 if (!attr) 869 panic("managed but no attr"); 870 #endif 871 if (flags & VM_PROT_ALL) 872 *attr |= PMAP_ATTR_REF; 873 if (flags & VM_PROT_WRITE) 874 *attr |= PMAP_ATTR_CHG; 875 } 876 877 s = splvm(); 878 879 /* Insert page into page table. */ 880 pte_enter(pm, va, tte); 881 882 /* If this is a real fault, enter it in the tlb */ 883 if (tte && ((flags & PMAP_WIRED) == 0)) { 884 ppc4xx_tlb_enter(pm->pm_ctx, va, tte); 885 } 886 splx(s); 887 888 /* Flush the real memory from the instruction cache. */ 889 if ((prot & VM_PROT_EXECUTE) && (tte & TTE_I) == 0) 890 __syncicache((void *)pa, PAGE_SIZE); 891 892 return 0; 893 } 894 895 void 896 pmap_unwire(struct pmap *pm, vaddr_t va) 897 { 898 struct pv_entry *pv; 899 paddr_t pa; 900 int s; 901 902 if (!pmap_extract(pm, va, &pa)) { 903 return; 904 } 905 906 pv = pa_to_pv(pa); 907 if (!pv) 908 return; 909 910 s = splvm(); 911 while (pv != NULL) { 912 if (pm == pv->pv_pm && PV_CMPVA(va, pv)) { 913 if (PV_ISWIRED(pv)) { 914 PV_UNWIRE(pv); 915 pm->pm_stats.wired_count--; 916 } 917 break; 918 } 919 pv = pv->pv_next; 920 } 921 splx(s); 922 } 923 924 void 925 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) 926 { 927 int s; 928 u_int tte; 929 struct pmap *pm = pmap_kernel(); 930 931 /* 932 * Have to remove any existing mapping first. 933 */ 934 935 /* 936 * Generate TTE. 937 * 938 * XXXX 939 * 940 * Since the kernel does not handle execution privileges properly, 941 * we will handle read and execute permissions together. 942 */ 943 tte = 0; 944 if (prot & VM_PROT_ALL) { 945 946 tte = TTE_PA(pa) | TTE_EX | TTE_ZONE(ZONE_PRIV); 947 /* XXXX -- need to support multiple page sizes. */ 948 tte |= TTE_SZ_16K; 949 #ifdef DIAGNOSTIC 950 if ((prot & (PME_NOCACHE | PME_WRITETHROUG)) == 951 (PME_NOCACHE | PME_WRITETHROUG)) 952 panic("pmap_kenter_pa: uncached & writethrough"); 953 #endif 954 if (prot & PME_NOCACHE) 955 /* Must be I/O mapping */ 956 tte |= TTE_I | TTE_G; 957 #ifdef PPC_4XX_NOCACHE 958 tte |= TTE_I; 959 #else 960 else if (prot & PME_WRITETHROUG) 961 /* Uncached and writethrough are not compatible */ 962 tte |= TTE_W; 963 #endif 964 if (prot & VM_PROT_WRITE) 965 tte |= TTE_WR; 966 } 967 968 s = splvm(); 969 970 /* Insert page into page table. */ 971 pte_enter(pm, va, tte); 972 splx(s); 973 } 974 975 void 976 pmap_kremove(vaddr_t va, vsize_t len) 977 { 978 979 while (len > 0) { 980 pte_enter(pmap_kernel(), va, 0); 981 va += PAGE_SIZE; 982 len -= PAGE_SIZE; 983 } 984 } 985 986 /* 987 * Remove the given range of mapping entries. 988 */ 989 void 990 pmap_remove(struct pmap *pm, vaddr_t va, vaddr_t endva) 991 { 992 int s; 993 paddr_t pa; 994 volatile u_int *ptp; 995 996 s = splvm(); 997 while (va < endva) { 998 999 if ((ptp = pte_find(pm, va)) && (pa = *ptp)) { 1000 pa = TTE_PA(pa); 1001 pmap_remove_pv(pm, va, pa); 1002 *ptp = 0; 1003 ppc4xx_tlb_flush(va, pm->pm_ctx); 1004 pm->pm_stats.resident_count--; 1005 } 1006 va += PAGE_SIZE; 1007 } 1008 1009 splx(s); 1010 } 1011 1012 /* 1013 * Get the physical page address for the given pmap/virtual address. 1014 */ 1015 boolean_t 1016 pmap_extract(struct pmap *pm, vaddr_t va, paddr_t *pap) 1017 { 1018 int seg = STIDX(va); 1019 int ptn = PTIDX(va); 1020 u_int pa = 0; 1021 int s; 1022 1023 s = splvm(); 1024 if (pm->pm_ptbl[seg] && (pa = pm->pm_ptbl[seg][ptn])) { 1025 *pap = TTE_PA(pa) | (va & PGOFSET); 1026 } 1027 splx(s); 1028 return (pa != 0); 1029 } 1030 1031 /* 1032 * Lower the protection on the specified range of this pmap. 1033 * 1034 * There are only two cases: either the protection is going to 0, 1035 * or it is going to read-only. 1036 */ 1037 void 1038 pmap_protect(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1039 { 1040 volatile u_int *ptp; 1041 int s, bic; 1042 1043 if ((prot & VM_PROT_READ) == 0) { 1044 pmap_remove(pm, sva, eva); 1045 return; 1046 } 1047 bic = 0; 1048 if ((prot & VM_PROT_WRITE) == 0) { 1049 bic |= TTE_WR; 1050 } 1051 if ((prot & VM_PROT_EXECUTE) == 0) { 1052 bic |= TTE_EX; 1053 } 1054 if (bic == 0) { 1055 return; 1056 } 1057 s = splvm(); 1058 while (sva < eva) { 1059 if ((ptp = pte_find(pm, sva)) != NULL) { 1060 *ptp &= ~bic; 1061 ppc4xx_tlb_flush(sva, pm->pm_ctx); 1062 } 1063 sva += PAGE_SIZE; 1064 } 1065 splx(s); 1066 } 1067 1068 boolean_t 1069 pmap_check_attr(struct vm_page *pg, u_int mask, int clear) 1070 { 1071 paddr_t pa; 1072 char *attr; 1073 int s, rv; 1074 1075 /* 1076 * First modify bits in cache. 1077 */ 1078 pa = VM_PAGE_TO_PHYS(pg); 1079 attr = pa_to_attr(pa); 1080 if (attr == NULL) 1081 return FALSE; 1082 1083 s = splvm(); 1084 rv = ((*attr & mask) != 0); 1085 if (clear) { 1086 *attr &= ~mask; 1087 pmap_page_protect(pg, mask == PMAP_ATTR_CHG ? VM_PROT_READ : 0); 1088 } 1089 splx(s); 1090 return rv; 1091 } 1092 1093 1094 /* 1095 * Lower the protection on the specified physical page. 1096 * 1097 * There are only two cases: either the protection is going to 0, 1098 * or it is going to read-only. 1099 */ 1100 void 1101 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 1102 { 1103 paddr_t pa = VM_PAGE_TO_PHYS(pg); 1104 vaddr_t va; 1105 struct pv_entry *pvh, *pv, *npv; 1106 struct pmap *pm; 1107 1108 pvh = pa_to_pv(pa); 1109 if (pvh == NULL) 1110 return; 1111 1112 /* Handle extra pvs which may be deleted in the operation */ 1113 for (pv = pvh->pv_next; pv; pv = npv) { 1114 npv = pv->pv_next; 1115 1116 pm = pv->pv_pm; 1117 va = pv->pv_va; 1118 pmap_protect(pm, va, va + PAGE_SIZE, prot); 1119 } 1120 /* Now check the head pv */ 1121 if (pvh->pv_pm) { 1122 pv = pvh; 1123 pm = pv->pv_pm; 1124 va = pv->pv_va; 1125 pmap_protect(pm, va, va + PAGE_SIZE, prot); 1126 } 1127 } 1128 1129 /* 1130 * Activate the address space for the specified process. If the process 1131 * is the current process, load the new MMU context. 1132 */ 1133 void 1134 pmap_activate(struct lwp *l) 1135 { 1136 #if 0 1137 struct pcb *pcb = &l->l_proc->p_addr->u_pcb; 1138 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 1139 1140 /* 1141 * XXX Normally performed in cpu_fork(). 1142 */ 1143 printf("pmap_activate(%p), pmap=%p\n",l,pmap); 1144 pcb->pcb_pm = pmap; 1145 #endif 1146 } 1147 1148 /* 1149 * Deactivate the specified process's address space. 1150 */ 1151 void 1152 pmap_deactivate(struct lwp *l) 1153 { 1154 } 1155 1156 /* 1157 * Synchronize caches corresponding to [addr, addr+len) in p. 1158 */ 1159 void 1160 pmap_procwr(struct proc *p, vaddr_t va, size_t len) 1161 { 1162 struct pmap *pm = p->p_vmspace->vm_map.pmap; 1163 int msr, ctx, opid, step; 1164 1165 step = CACHELINESIZE; 1166 1167 /* 1168 * Need to turn off IMMU and switch to user context. 1169 * (icbi uses DMMU). 1170 */ 1171 if (!(ctx = pm->pm_ctx)) { 1172 /* No context -- assign it one */ 1173 ctx_alloc(pm); 1174 ctx = pm->pm_ctx; 1175 } 1176 __asm volatile("mfmsr %0;" 1177 "li %1, %7;" 1178 "andc %1,%0,%1;" 1179 "mtmsr %1;" 1180 "sync;isync;" 1181 "mfpid %1;" 1182 "mtpid %2;" 1183 "sync; isync;" 1184 "1:" 1185 "dcbf 0,%3;" 1186 "icbi 0,%3;" 1187 "add %3,%3,%5;" 1188 "addc. %4,%4,%6;" 1189 "bge 1b;" 1190 "mtpid %1;" 1191 "mtmsr %0;" 1192 "sync; isync" 1193 : "=&r" (msr), "=&r" (opid) 1194 : "r" (ctx), "r" (va), "r" (len), "r" (step), "r" (-step), 1195 "K" (PSL_IR | PSL_DR)); 1196 } 1197 1198 1199 /* This has to be done in real mode !!! */ 1200 void 1201 ppc4xx_tlb_flush(vaddr_t va, int pid) 1202 { 1203 u_long i, found; 1204 u_long msr; 1205 1206 /* If there's no context then it can't be mapped. */ 1207 if (!pid) 1208 return; 1209 1210 __asm( "mfpid %1;" /* Save PID */ 1211 "mfmsr %2;" /* Save MSR */ 1212 "li %0,0;" /* Now clear MSR */ 1213 "mtmsr %0;" 1214 "mtpid %4;" /* Set PID */ 1215 "sync;" 1216 "tlbsx. %0,0,%3;" /* Search TLB */ 1217 "sync;" 1218 "mtpid %1;" /* Restore PID */ 1219 "mtmsr %2;" /* Restore MSR */ 1220 "sync;isync;" 1221 "li %1,1;" 1222 "beq 1f;" 1223 "li %1,0;" 1224 "1:" 1225 : "=&r" (i), "=&r" (found), "=&r" (msr) 1226 : "r" (va), "r" (pid)); 1227 if (found && !TLB_LOCKED(i)) { 1228 1229 /* Now flush translation */ 1230 __asm volatile( 1231 "tlbwe %0,%1,0;" 1232 "sync;isync;" 1233 : : "r" (0), "r" (i)); 1234 1235 tlb_info[i].ti_ctx = 0; 1236 tlb_info[i].ti_flags = 0; 1237 tlbnext = i; 1238 /* Successful flushes */ 1239 tlbflush_ev.ev_count++; 1240 } 1241 } 1242 1243 void 1244 ppc4xx_tlb_flush_all(void) 1245 { 1246 u_long i; 1247 1248 for (i = 0; i < NTLB; i++) 1249 if (!TLB_LOCKED(i)) { 1250 __asm volatile( 1251 "tlbwe %0,%1,0;" 1252 "sync;isync;" 1253 : : "r" (0), "r" (i)); 1254 tlb_info[i].ti_ctx = 0; 1255 tlb_info[i].ti_flags = 0; 1256 } 1257 1258 __asm volatile("sync;isync"); 1259 } 1260 1261 /* Find a TLB entry to evict. */ 1262 static int 1263 ppc4xx_tlb_find_victim(void) 1264 { 1265 int flags; 1266 1267 for (;;) { 1268 if (++tlbnext >= NTLB) 1269 tlbnext = tlb_nreserved; 1270 flags = tlb_info[tlbnext].ti_flags; 1271 if (!(flags & TLBF_USED) || 1272 (flags & (TLBF_LOCKED | TLBF_REF)) == 0) { 1273 u_long va, stack = (u_long)&va; 1274 1275 if (!((tlb_info[tlbnext].ti_va ^ stack) & (~PGOFSET)) && 1276 (tlb_info[tlbnext].ti_ctx == KERNEL_PID) && 1277 (flags & TLBF_USED)) { 1278 /* Kernel stack page */ 1279 flags |= TLBF_USED; 1280 tlb_info[tlbnext].ti_flags = flags; 1281 } else { 1282 /* Found it! */ 1283 return (tlbnext); 1284 } 1285 } else { 1286 tlb_info[tlbnext].ti_flags = (flags & ~TLBF_REF); 1287 } 1288 } 1289 } 1290 1291 void 1292 ppc4xx_tlb_enter(int ctx, vaddr_t va, u_int pte) 1293 { 1294 u_long th, tl, idx; 1295 tlbpid_t pid; 1296 u_short msr; 1297 paddr_t pa; 1298 int s, sz; 1299 1300 tlbenter_ev.ev_count++; 1301 1302 sz = (pte & TTE_SZ_MASK) >> TTE_SZ_SHIFT; 1303 pa = (pte & TTE_RPN_MASK(sz)); 1304 th = (va & TLB_EPN_MASK) | (sz << TLB_SIZE_SHFT) | TLB_VALID; 1305 tl = (pte & ~TLB_RPN_MASK) | pa; 1306 tl |= ppc4xx_tlbflags(va, pa); 1307 1308 s = splhigh(); 1309 idx = ppc4xx_tlb_find_victim(); 1310 1311 #ifdef DIAGNOSTIC 1312 if ((idx < tlb_nreserved) || (idx >= NTLB)) { 1313 panic("ppc4xx_tlb_enter: replacing entry %ld", idx); 1314 } 1315 #endif 1316 1317 tlb_info[idx].ti_va = (va & TLB_EPN_MASK); 1318 tlb_info[idx].ti_ctx = ctx; 1319 tlb_info[idx].ti_flags = TLBF_USED | TLBF_REF; 1320 1321 __asm volatile( 1322 "mfmsr %0;" /* Save MSR */ 1323 "li %1,0;" 1324 "tlbwe %1,%3,0;" /* Invalidate old entry. */ 1325 "mtmsr %1;" /* Clear MSR */ 1326 "mfpid %1;" /* Save old PID */ 1327 "mtpid %2;" /* Load translation ctx */ 1328 "sync; isync;" 1329 #ifdef DEBUG 1330 "andi. %3,%3,63;" 1331 "tweqi %3,0;" /* XXXXX DEBUG trap on index 0 */ 1332 #endif 1333 "tlbwe %4,%3,1; tlbwe %5,%3,0;" /* Set TLB */ 1334 "sync; isync;" 1335 "mtpid %1; mtmsr %0;" /* Restore PID and MSR */ 1336 "sync; isync;" 1337 : "=&r" (msr), "=&r" (pid) 1338 : "r" (ctx), "r" (idx), "r" (tl), "r" (th)); 1339 splx(s); 1340 } 1341 1342 void 1343 ppc4xx_tlb_init(void) 1344 { 1345 int i; 1346 1347 /* Mark reserved TLB entries */ 1348 for (i = 0; i < tlb_nreserved; i++) { 1349 tlb_info[i].ti_flags = TLBF_LOCKED | TLBF_USED; 1350 tlb_info[i].ti_ctx = KERNEL_PID; 1351 } 1352 1353 /* Setup security zones */ 1354 /* Z0 - accessible by kernel only if TLB entry permissions allow 1355 * Z1,Z2 - access is controlled by TLB entry permissions 1356 * Z3 - full access regardless of TLB entry permissions 1357 */ 1358 1359 __asm volatile( 1360 "mtspr %0,%1;" 1361 "sync;" 1362 :: "K"(SPR_ZPR), "r" (0x1b000000)); 1363 } 1364 1365 /* 1366 * ppc4xx_tlb_size_mask: 1367 * 1368 * Roundup size to supported page size, return TLBHI mask and real size. 1369 */ 1370 static int 1371 ppc4xx_tlb_size_mask(size_t size, int *mask, int *rsiz) 1372 { 1373 int i; 1374 1375 for (i = 0; i < __arraycount(tlbsize); i++) 1376 if (size <= tlbsize[i]) { 1377 *mask = (i << TLB_SIZE_SHFT); 1378 *rsiz = tlbsize[i]; 1379 return (0); 1380 } 1381 return (EINVAL); 1382 } 1383 1384 /* 1385 * ppc4xx_tlb_mapiodev: 1386 * 1387 * Lookup virtual address of mapping previously entered via 1388 * ppc4xx_tlb_reserve. Search TLB directly so that we don't 1389 * need to waste extra storage for reserved mappings. Note 1390 * that reading TLBHI also sets PID, but all reserved mappings 1391 * use KERNEL_PID, so the side effect is nil. 1392 */ 1393 void * 1394 ppc4xx_tlb_mapiodev(paddr_t base, psize_t len) 1395 { 1396 paddr_t pa; 1397 vaddr_t va; 1398 u_int lo, hi, sz; 1399 int i; 1400 1401 /* tlb_nreserved is only allowed to grow, so this is safe. */ 1402 for (i = 0; i < tlb_nreserved; i++) { 1403 __asm volatile ( 1404 " tlbre %0,%2,1 \n" /* TLBLO */ 1405 " tlbre %1,%2,0 \n" /* TLBHI */ 1406 : "=&r" (lo), "=&r" (hi) 1407 : "r" (i)); 1408 1409 KASSERT(hi & TLB_VALID); 1410 KASSERT(mfspr(SPR_PID) == KERNEL_PID); 1411 1412 pa = (lo & TLB_RPN_MASK); 1413 if (base < pa) 1414 continue; 1415 1416 sz = tlbsize[(hi & TLB_SIZE_MASK) >> TLB_SIZE_SHFT]; 1417 if ((base + len) > (pa + sz)) 1418 continue; 1419 1420 va = (hi & TLB_EPN_MASK) + (base & (sz - 1)); /* sz = 2^n */ 1421 return (void *)(va); 1422 } 1423 1424 return (NULL); 1425 } 1426 1427 /* 1428 * ppc4xx_tlb_reserve: 1429 * 1430 * Map physical range to kernel virtual chunk via reserved TLB entry. 1431 */ 1432 void 1433 ppc4xx_tlb_reserve(paddr_t pa, vaddr_t va, size_t size, int flags) 1434 { 1435 u_int lo, hi; 1436 int szmask, rsize; 1437 1438 /* Called before pmap_bootstrap(), va outside kernel space. */ 1439 KASSERT(va < VM_MIN_KERNEL_ADDRESS || va >= VM_MAX_KERNEL_ADDRESS); 1440 KASSERT(! pmap_bootstrap_done); 1441 KASSERT(tlb_nreserved < NTLB); 1442 1443 /* Resolve size. */ 1444 if (ppc4xx_tlb_size_mask(size, &szmask, &rsize) != 0) 1445 panic("ppc4xx_tlb_reserve: entry %d, %zuB too large", 1446 size, tlb_nreserved); 1447 1448 /* Real size will be power of two >= 1024, so this is OK. */ 1449 pa &= ~(rsize - 1); /* RPN */ 1450 va &= ~(rsize - 1); /* EPN */ 1451 1452 lo = pa | TLB_WR | flags; 1453 hi = va | TLB_VALID | szmask; 1454 1455 #ifdef PPC_4XX_NOCACHE 1456 lo |= TLB_I; 1457 #endif 1458 1459 __asm volatile( 1460 " tlbwe %1,%0,1 \n" /* write TLBLO */ 1461 " tlbwe %2,%0,0 \n" /* write TLBHI */ 1462 " sync \n" 1463 " isync \n" 1464 : : "r" (tlb_nreserved), "r" (lo), "r" (hi)); 1465 1466 tlb_nreserved++; 1467 } 1468 1469 /* 1470 * We should pass the ctx in from trap code. 1471 */ 1472 int 1473 pmap_tlbmiss(vaddr_t va, int ctx) 1474 { 1475 volatile u_int *pte; 1476 u_long tte; 1477 1478 tlbmiss_ev.ev_count++; 1479 1480 /* 1481 * XXXX We will reserve 0-0x80000000 for va==pa mappings. 1482 */ 1483 if (ctx != KERNEL_PID || (va & 0x80000000)) { 1484 pte = pte_find((struct pmap *)__UNVOLATILE(ctxbusy[ctx]), va); 1485 if (pte == NULL) { 1486 /* Map unmanaged addresses directly for kernel access */ 1487 return 1; 1488 } 1489 tte = *pte; 1490 if (tte == 0) { 1491 return 1; 1492 } 1493 } else { 1494 /* Create a 16MB writable mapping. */ 1495 #ifdef PPC_4XX_NOCACHE 1496 tte = TTE_PA(va) | TTE_ZONE(ZONE_PRIV) | TTE_SZ_16M | TTE_I | TTE_WR; 1497 #else 1498 tte = TTE_PA(va) | TTE_ZONE(ZONE_PRIV) | TTE_SZ_16M | TTE_WR; 1499 #endif 1500 } 1501 tlbhit_ev.ev_count++; 1502 ppc4xx_tlb_enter(ctx, va, tte); 1503 1504 return 0; 1505 } 1506 1507 /* 1508 * Flush all the entries matching a context from the TLB. 1509 */ 1510 static int 1511 ctx_flush(int cnum) 1512 { 1513 int i; 1514 1515 /* We gotta steal this context */ 1516 for (i = tlb_nreserved; i < NTLB; i++) { 1517 if (tlb_info[i].ti_ctx == cnum) { 1518 /* Can't steal ctx if it has a locked entry. */ 1519 if (TLB_LOCKED(i)) { 1520 #ifdef DIAGNOSTIC 1521 printf("ctx_flush: can't invalidate " 1522 "locked mapping %d " 1523 "for context %d\n", i, cnum); 1524 #ifdef DDB 1525 Debugger(); 1526 #endif 1527 #endif 1528 return (1); 1529 } 1530 #ifdef DIAGNOSTIC 1531 if (i < tlb_nreserved) 1532 panic("TLB entry %d not locked", i); 1533 #endif 1534 /* Invalidate particular TLB entry regardless of locked status */ 1535 __asm volatile("tlbwe %0,%1,0" : :"r"(0),"r"(i)); 1536 tlb_info[i].ti_flags = 0; 1537 } 1538 } 1539 return (0); 1540 } 1541 1542 /* 1543 * Allocate a context. If necessary, steal one from someone else. 1544 * 1545 * The new context is flushed from the TLB before returning. 1546 */ 1547 int 1548 ctx_alloc(struct pmap *pm) 1549 { 1550 int s, cnum; 1551 static int next = MINCTX; 1552 1553 if (pm == pmap_kernel()) { 1554 #ifdef DIAGNOSTIC 1555 printf("ctx_alloc: kernel pmap!\n"); 1556 #endif 1557 return (0); 1558 } 1559 s = splvm(); 1560 1561 /* Find a likely context. */ 1562 cnum = next; 1563 do { 1564 if ((++cnum) > NUMCTX) 1565 cnum = MINCTX; 1566 } while (ctxbusy[cnum] != NULL && cnum != next); 1567 1568 /* Now clean it out */ 1569 oops: 1570 if (cnum < MINCTX) 1571 cnum = MINCTX; /* Never steal ctx 0 or 1 */ 1572 if (ctx_flush(cnum)) { 1573 /* oops -- something's wired. */ 1574 if ((++cnum) > NUMCTX) 1575 cnum = MINCTX; 1576 goto oops; 1577 } 1578 1579 if (ctxbusy[cnum]) { 1580 #ifdef DEBUG 1581 /* We should identify this pmap and clear it */ 1582 printf("Warning: stealing context %d\n", cnum); 1583 #endif 1584 ctxbusy[cnum]->pm_ctx = 0; 1585 } 1586 ctxbusy[cnum] = pm; 1587 next = cnum; 1588 splx(s); 1589 pm->pm_ctx = cnum; 1590 1591 return cnum; 1592 } 1593 1594 /* 1595 * Give away a context. 1596 */ 1597 void 1598 ctx_free(struct pmap *pm) 1599 { 1600 int oldctx; 1601 1602 oldctx = pm->pm_ctx; 1603 1604 if (oldctx == 0) 1605 panic("ctx_free: freeing kernel context"); 1606 #ifdef DIAGNOSTIC 1607 if (ctxbusy[oldctx] == 0) 1608 printf("ctx_free: freeing free context %d\n", oldctx); 1609 if (ctxbusy[oldctx] != pm) { 1610 printf("ctx_free: freeing someone esle's context\n " 1611 "ctxbusy[%d] = %p, pm->pm_ctx = %p\n", 1612 oldctx, (void *)(u_long)ctxbusy[oldctx], pm); 1613 #ifdef DDB 1614 Debugger(); 1615 #endif 1616 } 1617 #endif 1618 /* We should verify it has not been stolen and reallocated... */ 1619 ctxbusy[oldctx] = NULL; 1620 ctx_flush(oldctx); 1621 } 1622 1623 1624 #ifdef DEBUG 1625 /* 1626 * Test ref/modify handling. 1627 */ 1628 void pmap_testout __P((void)); 1629 void 1630 pmap_testout() 1631 { 1632 vaddr_t va; 1633 volatile int *loc; 1634 int val = 0; 1635 paddr_t pa; 1636 struct vm_page *pg; 1637 int ref, mod; 1638 1639 /* Allocate a page */ 1640 va = (vaddr_t)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 1641 UVM_KMF_WIRED | UVM_KMF_ZERO); 1642 loc = (int*)va; 1643 1644 pmap_extract(pmap_kernel(), va, &pa); 1645 pg = PHYS_TO_VM_PAGE(pa); 1646 pmap_unwire(pmap_kernel(), va); 1647 1648 pmap_kremove(va, PAGE_SIZE); 1649 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1650 pmap_update(pmap_kernel()); 1651 1652 /* Now clear reference and modify */ 1653 ref = pmap_clear_reference(pg); 1654 mod = pmap_clear_modify(pg); 1655 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1656 (void *)(u_long)va, (long)pa, 1657 ref, mod); 1658 1659 /* Check it's properly cleared */ 1660 ref = pmap_is_referenced(pg); 1661 mod = pmap_is_modified(pg); 1662 printf("Checking cleared page: ref %d, mod %d\n", 1663 ref, mod); 1664 1665 /* Reference page */ 1666 val = *loc; 1667 1668 ref = pmap_is_referenced(pg); 1669 mod = pmap_is_modified(pg); 1670 printf("Referenced page: ref %d, mod %d val %x\n", 1671 ref, mod, val); 1672 1673 /* Now clear reference and modify */ 1674 ref = pmap_clear_reference(pg); 1675 mod = pmap_clear_modify(pg); 1676 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1677 (void *)(u_long)va, (long)pa, 1678 ref, mod); 1679 1680 /* Modify page */ 1681 *loc = 1; 1682 1683 ref = pmap_is_referenced(pg); 1684 mod = pmap_is_modified(pg); 1685 printf("Modified page: ref %d, mod %d\n", 1686 ref, mod); 1687 1688 /* Now clear reference and modify */ 1689 ref = pmap_clear_reference(pg); 1690 mod = pmap_clear_modify(pg); 1691 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1692 (void *)(u_long)va, (long)pa, 1693 ref, mod); 1694 1695 /* Check it's properly cleared */ 1696 ref = pmap_is_referenced(pg); 1697 mod = pmap_is_modified(pg); 1698 printf("Checking cleared page: ref %d, mod %d\n", 1699 ref, mod); 1700 1701 /* Modify page */ 1702 *loc = 1; 1703 1704 ref = pmap_is_referenced(pg); 1705 mod = pmap_is_modified(pg); 1706 printf("Modified page: ref %d, mod %d\n", 1707 ref, mod); 1708 1709 /* Check pmap_protect() */ 1710 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_READ); 1711 pmap_update(pmap_kernel()); 1712 ref = pmap_is_referenced(pg); 1713 mod = pmap_is_modified(pg); 1714 printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n", 1715 ref, mod); 1716 1717 /* Now clear reference and modify */ 1718 ref = pmap_clear_reference(pg); 1719 mod = pmap_clear_modify(pg); 1720 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1721 (void *)(u_long)va, (long)pa, 1722 ref, mod); 1723 1724 /* Reference page */ 1725 val = *loc; 1726 1727 ref = pmap_is_referenced(pg); 1728 mod = pmap_is_modified(pg); 1729 printf("Referenced page: ref %d, mod %d val %x\n", 1730 ref, mod, val); 1731 1732 /* Now clear reference and modify */ 1733 ref = pmap_clear_reference(pg); 1734 mod = pmap_clear_modify(pg); 1735 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1736 (void *)(u_long)va, (long)pa, 1737 ref, mod); 1738 1739 /* Modify page */ 1740 #if 0 1741 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1742 pmap_update(pmap_kernel()); 1743 #endif 1744 *loc = 1; 1745 1746 ref = pmap_is_referenced(pg); 1747 mod = pmap_is_modified(pg); 1748 printf("Modified page: ref %d, mod %d\n", 1749 ref, mod); 1750 1751 /* Check pmap_protect() */ 1752 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_NONE); 1753 pmap_update(pmap_kernel()); 1754 ref = pmap_is_referenced(pg); 1755 mod = pmap_is_modified(pg); 1756 printf("pmap_protect(): ref %d, mod %d\n", 1757 ref, mod); 1758 1759 /* Now clear reference and modify */ 1760 ref = pmap_clear_reference(pg); 1761 mod = pmap_clear_modify(pg); 1762 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1763 (void *)(u_long)va, (long)pa, 1764 ref, mod); 1765 1766 /* Reference page */ 1767 val = *loc; 1768 1769 ref = pmap_is_referenced(pg); 1770 mod = pmap_is_modified(pg); 1771 printf("Referenced page: ref %d, mod %d val %x\n", 1772 ref, mod, val); 1773 1774 /* Now clear reference and modify */ 1775 ref = pmap_clear_reference(pg); 1776 mod = pmap_clear_modify(pg); 1777 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1778 (void *)(u_long)va, (long)pa, 1779 ref, mod); 1780 1781 /* Modify page */ 1782 #if 0 1783 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1784 pmap_update(pmap_kernel()); 1785 #endif 1786 *loc = 1; 1787 1788 ref = pmap_is_referenced(pg); 1789 mod = pmap_is_modified(pg); 1790 printf("Modified page: ref %d, mod %d\n", 1791 ref, mod); 1792 1793 /* Check pmap_pag_protect() */ 1794 pmap_page_protect(pg, VM_PROT_READ); 1795 ref = pmap_is_referenced(pg); 1796 mod = pmap_is_modified(pg); 1797 printf("pmap_page_protect(VM_PROT_READ): ref %d, mod %d\n", 1798 ref, mod); 1799 1800 /* Now clear reference and modify */ 1801 ref = pmap_clear_reference(pg); 1802 mod = pmap_clear_modify(pg); 1803 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1804 (void *)(u_long)va, (long)pa, 1805 ref, mod); 1806 1807 /* Reference page */ 1808 val = *loc; 1809 1810 ref = pmap_is_referenced(pg); 1811 mod = pmap_is_modified(pg); 1812 printf("Referenced page: ref %d, mod %d val %x\n", 1813 ref, mod, val); 1814 1815 /* Now clear reference and modify */ 1816 ref = pmap_clear_reference(pg); 1817 mod = pmap_clear_modify(pg); 1818 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1819 (void *)(u_long)va, (long)pa, 1820 ref, mod); 1821 1822 /* Modify page */ 1823 #if 0 1824 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1825 pmap_update(pmap_kernel()); 1826 #endif 1827 *loc = 1; 1828 1829 ref = pmap_is_referenced(pg); 1830 mod = pmap_is_modified(pg); 1831 printf("Modified page: ref %d, mod %d\n", 1832 ref, mod); 1833 1834 /* Check pmap_pag_protect() */ 1835 pmap_page_protect(pg, VM_PROT_NONE); 1836 ref = pmap_is_referenced(pg); 1837 mod = pmap_is_modified(pg); 1838 printf("pmap_page_protect(): ref %d, mod %d\n", 1839 ref, mod); 1840 1841 /* Now clear reference and modify */ 1842 ref = pmap_clear_reference(pg); 1843 mod = pmap_clear_modify(pg); 1844 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1845 (void *)(u_long)va, (long)pa, 1846 ref, mod); 1847 1848 1849 /* Reference page */ 1850 val = *loc; 1851 1852 ref = pmap_is_referenced(pg); 1853 mod = pmap_is_modified(pg); 1854 printf("Referenced page: ref %d, mod %d val %x\n", 1855 ref, mod, val); 1856 1857 /* Now clear reference and modify */ 1858 ref = pmap_clear_reference(pg); 1859 mod = pmap_clear_modify(pg); 1860 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1861 (void *)(u_long)va, (long)pa, 1862 ref, mod); 1863 1864 /* Modify page */ 1865 #if 0 1866 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1867 pmap_update(pmap_kernel()); 1868 #endif 1869 *loc = 1; 1870 1871 ref = pmap_is_referenced(pg); 1872 mod = pmap_is_modified(pg); 1873 printf("Modified page: ref %d, mod %d\n", 1874 ref, mod); 1875 1876 /* Unmap page */ 1877 pmap_remove(pmap_kernel(), va, va+1); 1878 pmap_update(pmap_kernel()); 1879 ref = pmap_is_referenced(pg); 1880 mod = pmap_is_modified(pg); 1881 printf("Unmapped page: ref %d, mod %d\n", ref, mod); 1882 1883 /* Now clear reference and modify */ 1884 ref = pmap_clear_reference(pg); 1885 mod = pmap_clear_modify(pg); 1886 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1887 (void *)(u_long)va, (long)pa, ref, mod); 1888 1889 /* Check it's properly cleared */ 1890 ref = pmap_is_referenced(pg); 1891 mod = pmap_is_modified(pg); 1892 printf("Checking cleared page: ref %d, mod %d\n", 1893 ref, mod); 1894 1895 pmap_remove(pmap_kernel(), va, va + PAGE_SIZE); 1896 pmap_kenter_pa(va, pa, VM_PROT_ALL); 1897 uvm_km_free(kernel_map, (vaddr_t)va, PAGE_SIZE, UVM_KMF_WIRED); 1898 } 1899 #endif 1900