1 /* $NetBSD: pmap.c,v 1.54 2009/03/18 10:22:34 cegger Exp $ */ 2 3 /* 4 * Copyright 2001 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Eduardo Horvath and Simon Burge for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 40 * Copyright (C) 1995, 1996 TooLs GmbH. 41 * All rights reserved. 42 * 43 * Redistribution and use in source and binary forms, with or without 44 * modification, are permitted provided that the following conditions 45 * are met: 46 * 1. Redistributions of source code must retain the above copyright 47 * notice, this list of conditions and the following disclaimer. 48 * 2. Redistributions in binary form must reproduce the above copyright 49 * notice, this list of conditions and the following disclaimer in the 50 * documentation and/or other materials provided with the distribution. 51 * 3. All advertising materials mentioning features or use of this software 52 * must display the following acknowledgement: 53 * This product includes software developed by TooLs GmbH. 54 * 4. The name of TooLs GmbH may not be used to endorse or promote products 55 * derived from this software without specific prior written permission. 56 * 57 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 58 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 59 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 60 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 61 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 62 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 63 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 64 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 65 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 66 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 67 */ 68 69 #include <sys/cdefs.h> 70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.54 2009/03/18 10:22:34 cegger Exp $"); 71 72 #include <sys/param.h> 73 #include <sys/malloc.h> 74 #include <sys/proc.h> 75 #include <sys/user.h> 76 #include <sys/queue.h> 77 #include <sys/systm.h> 78 #include <sys/pool.h> 79 #include <sys/device.h> 80 81 #include <uvm/uvm.h> 82 83 #include <machine/cpu.h> 84 #include <machine/pcb.h> 85 #include <machine/powerpc.h> 86 87 #include <powerpc/spr.h> 88 #include <machine/tlb.h> 89 90 /* 91 * kernmap is an array of PTEs large enough to map in 92 * 4GB. At 16KB/page it is 256K entries or 2MB. 93 */ 94 #define KERNMAP_SIZE ((0xffffffffU/PAGE_SIZE)+1) 95 void *kernmap; 96 97 #define MINCTX 2 98 #define NUMCTX 256 99 100 volatile struct pmap *ctxbusy[NUMCTX]; 101 102 #define TLBF_USED 0x1 103 #define TLBF_REF 0x2 104 #define TLBF_LOCKED 0x4 105 #define TLB_LOCKED(i) (tlb_info[(i)].ti_flags & TLBF_LOCKED) 106 107 typedef struct tlb_info_s { 108 char ti_flags; 109 char ti_ctx; /* TLB_PID assiciated with the entry */ 110 u_int ti_va; 111 } tlb_info_t; 112 113 volatile tlb_info_t tlb_info[NTLB]; 114 /* We'll use a modified FIFO replacement policy cause it's cheap */ 115 volatile int tlbnext; 116 117 static int tlb_nreserved = 0; 118 static int pmap_bootstrap_done = 0; 119 120 /* Event counters */ 121 struct evcnt tlbmiss_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 122 NULL, "cpu", "tlbmiss"); 123 struct evcnt tlbhit_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 124 NULL, "cpu", "tlbhit"); 125 struct evcnt tlbflush_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 126 NULL, "cpu", "tlbflush"); 127 struct evcnt tlbenter_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 128 NULL, "cpu", "tlbenter"); 129 130 struct pmap kernel_pmap_; 131 struct pmap *const kernel_pmap_ptr = &kernel_pmap_; 132 133 int physmem; 134 static int npgs; 135 static u_int nextavail; 136 #ifndef MSGBUFADDR 137 extern paddr_t msgbuf_paddr; 138 #endif 139 140 static struct mem_region *mem, *avail; 141 142 /* 143 * This is a cache of referenced/modified bits. 144 * Bits herein are shifted by ATTRSHFT. 145 */ 146 static char *pmap_attrib; 147 148 #define PV_WIRED 0x1 149 #define PV_WIRE(pv) ((pv)->pv_va |= PV_WIRED) 150 #define PV_UNWIRE(pv) ((pv)->pv_va &= ~PV_WIRED) 151 #define PV_ISWIRED(pv) ((pv)->pv_va & PV_WIRED) 152 #define PV_CMPVA(va,pv) (!(((pv)->pv_va ^ (va)) & (~PV_WIRED))) 153 154 struct pv_entry { 155 struct pv_entry *pv_next; /* Linked list of mappings */ 156 vaddr_t pv_va; /* virtual address of mapping */ 157 struct pmap *pv_pm; 158 }; 159 160 /* Each index corresponds to TLB_SIZE_* value. */ 161 static size_t tlbsize[] = { 162 1024, /* TLB_SIZE_1K */ 163 4096, /* TLB_SIZE_4K */ 164 16384, /* TLB_SIZE_16K */ 165 65536, /* TLB_SIZE_64K */ 166 262144, /* TLB_SIZE_256K */ 167 1048576, /* TLB_SIZE_1M */ 168 4194304, /* TLB_SIZE_4M */ 169 16777216, /* TLB_SIZE_16M */ 170 }; 171 172 struct pv_entry *pv_table; 173 static struct pool pv_pool; 174 175 static int pmap_initialized; 176 177 static int ctx_flush(int); 178 179 inline struct pv_entry *pa_to_pv(paddr_t); 180 static inline char *pa_to_attr(paddr_t); 181 182 static inline volatile u_int *pte_find(struct pmap *, vaddr_t); 183 static inline int pte_enter(struct pmap *, vaddr_t, u_int); 184 185 static inline int pmap_enter_pv(struct pmap *, vaddr_t, paddr_t, int); 186 static void pmap_remove_pv(struct pmap *, vaddr_t, paddr_t); 187 188 static int ppc4xx_tlb_size_mask(size_t, int *, int *); 189 190 191 inline struct pv_entry * 192 pa_to_pv(paddr_t pa) 193 { 194 int bank, pg; 195 196 bank = vm_physseg_find(atop(pa), &pg); 197 if (bank == -1) 198 return NULL; 199 return &vm_physmem[bank].pmseg.pvent[pg]; 200 } 201 202 static inline char * 203 pa_to_attr(paddr_t pa) 204 { 205 int bank, pg; 206 207 bank = vm_physseg_find(atop(pa), &pg); 208 if (bank == -1) 209 return NULL; 210 return &vm_physmem[bank].pmseg.attrs[pg]; 211 } 212 213 /* 214 * Insert PTE into page table. 215 */ 216 int 217 pte_enter(struct pmap *pm, vaddr_t va, u_int pte) 218 { 219 int seg = STIDX(va); 220 int ptn = PTIDX(va); 221 u_int oldpte; 222 223 if (!pm->pm_ptbl[seg]) { 224 /* Don't allocate a page to clear a non-existent mapping. */ 225 if (!pte) 226 return (0); 227 /* Allocate a page XXXX this will sleep! */ 228 pm->pm_ptbl[seg] = 229 (uint *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 230 UVM_KMF_WIRED | UVM_KMF_ZERO); 231 } 232 oldpte = pm->pm_ptbl[seg][ptn]; 233 pm->pm_ptbl[seg][ptn] = pte; 234 235 /* Flush entry. */ 236 ppc4xx_tlb_flush(va, pm->pm_ctx); 237 if (oldpte != pte) { 238 if (pte == 0) 239 pm->pm_stats.resident_count--; 240 else 241 pm->pm_stats.resident_count++; 242 } 243 return (1); 244 } 245 246 /* 247 * Get a pointer to a PTE in a page table. 248 */ 249 volatile u_int * 250 pte_find(struct pmap *pm, vaddr_t va) 251 { 252 int seg = STIDX(va); 253 int ptn = PTIDX(va); 254 255 if (pm->pm_ptbl[seg]) 256 return (&pm->pm_ptbl[seg][ptn]); 257 258 return (NULL); 259 } 260 261 /* 262 * This is called during initppc, before the system is really initialized. 263 */ 264 void 265 pmap_bootstrap(u_int kernelstart, u_int kernelend) 266 { 267 struct mem_region *mp, *mp1; 268 int cnt, i; 269 u_int s, e, sz; 270 271 tlbnext = tlb_nreserved; 272 273 /* 274 * Allocate the kernel page table at the end of 275 * kernel space so it's in the locked TTE. 276 */ 277 kernmap = (void *)kernelend; 278 279 /* 280 * Initialize kernel page table. 281 */ 282 for (i = 0; i < STSZ; i++) { 283 pmap_kernel()->pm_ptbl[i] = 0; 284 } 285 ctxbusy[0] = ctxbusy[1] = pmap_kernel(); 286 287 /* 288 * Announce page-size to the VM-system 289 */ 290 uvmexp.pagesize = NBPG; 291 uvm_setpagesize(); 292 293 /* 294 * Get memory. 295 */ 296 mem_regions(&mem, &avail); 297 for (mp = mem; mp->size; mp++) { 298 physmem += btoc(mp->size); 299 printf("+%lx,",mp->size); 300 } 301 printf("\n"); 302 ppc4xx_tlb_init(); 303 /* 304 * Count the number of available entries. 305 */ 306 for (cnt = 0, mp = avail; mp->size; mp++) 307 cnt++; 308 309 /* 310 * Page align all regions. 311 * Non-page aligned memory isn't very interesting to us. 312 * Also, sort the entries for ascending addresses. 313 */ 314 kernelstart &= ~PGOFSET; 315 kernelend = (kernelend + PGOFSET) & ~PGOFSET; 316 for (mp = avail; mp->size; mp++) { 317 s = mp->start; 318 e = mp->start + mp->size; 319 printf("%08x-%08x -> ",s,e); 320 /* 321 * Check whether this region holds all of the kernel. 322 */ 323 if (s < kernelstart && e > kernelend) { 324 avail[cnt].start = kernelend; 325 avail[cnt++].size = e - kernelend; 326 e = kernelstart; 327 } 328 /* 329 * Look whether this regions starts within the kernel. 330 */ 331 if (s >= kernelstart && s < kernelend) { 332 if (e <= kernelend) 333 goto empty; 334 s = kernelend; 335 } 336 /* 337 * Now look whether this region ends within the kernel. 338 */ 339 if (e > kernelstart && e <= kernelend) { 340 if (s >= kernelstart) 341 goto empty; 342 e = kernelstart; 343 } 344 /* 345 * Now page align the start and size of the region. 346 */ 347 s = round_page(s); 348 e = trunc_page(e); 349 if (e < s) 350 e = s; 351 sz = e - s; 352 printf("%08x-%08x = %x\n",s,e,sz); 353 /* 354 * Check whether some memory is left here. 355 */ 356 if (sz == 0) { 357 empty: 358 memmove(mp, mp + 1, 359 (cnt - (mp - avail)) * sizeof *mp); 360 cnt--; 361 mp--; 362 continue; 363 } 364 /* 365 * Do an insertion sort. 366 */ 367 npgs += btoc(sz); 368 for (mp1 = avail; mp1 < mp; mp1++) 369 if (s < mp1->start) 370 break; 371 if (mp1 < mp) { 372 memmove(mp1 + 1, mp1, (char *)mp - (char *)mp1); 373 mp1->start = s; 374 mp1->size = sz; 375 } else { 376 mp->start = s; 377 mp->size = sz; 378 } 379 } 380 381 /* 382 * We cannot do pmap_steal_memory here, 383 * since we don't run with translation enabled yet. 384 */ 385 #ifndef MSGBUFADDR 386 /* 387 * allow for msgbuf 388 */ 389 sz = round_page(MSGBUFSIZE); 390 mp = NULL; 391 for (mp1 = avail; mp1->size; mp1++) 392 if (mp1->size >= sz) 393 mp = mp1; 394 if (mp == NULL) 395 panic("not enough memory?"); 396 397 npgs -= btoc(sz); 398 msgbuf_paddr = mp->start + mp->size - sz; 399 mp->size -= sz; 400 if (mp->size <= 0) 401 memmove(mp, mp + 1, (cnt - (mp - avail)) * sizeof *mp); 402 #endif 403 404 for (mp = avail; mp->size; mp++) 405 uvm_page_physload(atop(mp->start), atop(mp->start + mp->size), 406 atop(mp->start), atop(mp->start + mp->size), 407 VM_FREELIST_DEFAULT); 408 409 /* 410 * Initialize kernel pmap and hardware. 411 */ 412 /* Setup TLB pid allocator so it knows we alreadu using PID 1 */ 413 pmap_kernel()->pm_ctx = KERNEL_PID; 414 nextavail = avail->start; 415 416 evcnt_attach_static(&tlbmiss_ev); 417 evcnt_attach_static(&tlbhit_ev); 418 evcnt_attach_static(&tlbflush_ev); 419 evcnt_attach_static(&tlbenter_ev); 420 421 pmap_bootstrap_done = 1; 422 } 423 424 /* 425 * Restrict given range to physical memory 426 * 427 * (Used by /dev/mem) 428 */ 429 void 430 pmap_real_memory(paddr_t *start, psize_t *size) 431 { 432 struct mem_region *mp; 433 434 for (mp = mem; mp->size; mp++) { 435 if (*start + *size > mp->start && 436 *start < mp->start + mp->size) { 437 if (*start < mp->start) { 438 *size -= mp->start - *start; 439 *start = mp->start; 440 } 441 if (*start + *size > mp->start + mp->size) 442 *size = mp->start + mp->size - *start; 443 return; 444 } 445 } 446 *size = 0; 447 } 448 449 /* 450 * Initialize anything else for pmap handling. 451 * Called during vm_init(). 452 */ 453 void 454 pmap_init(void) 455 { 456 struct pv_entry *pv; 457 vsize_t sz; 458 vaddr_t addr; 459 int i, s; 460 int bank; 461 char *attr; 462 463 sz = (vsize_t)((sizeof(struct pv_entry) + 1) * npgs); 464 sz = round_page(sz); 465 addr = uvm_km_alloc(kernel_map, sz, 0, UVM_KMF_WIRED | UVM_KMF_ZERO); 466 s = splvm(); 467 pv = pv_table = (struct pv_entry *)addr; 468 for (i = npgs; --i >= 0;) 469 pv++->pv_pm = NULL; 470 pmap_attrib = (char *)pv; 471 memset(pv, 0, npgs); 472 473 pv = pv_table; 474 attr = pmap_attrib; 475 for (bank = 0; bank < vm_nphysseg; bank++) { 476 sz = vm_physmem[bank].end - vm_physmem[bank].start; 477 vm_physmem[bank].pmseg.pvent = pv; 478 vm_physmem[bank].pmseg.attrs = attr; 479 pv += sz; 480 attr += sz; 481 } 482 483 pmap_initialized = 1; 484 splx(s); 485 486 /* Setup a pool for additional pvlist structures */ 487 pool_init(&pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pv_entry", NULL, 488 IPL_VM); 489 } 490 491 /* 492 * How much virtual space is available to the kernel? 493 */ 494 void 495 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 496 { 497 498 #if 0 499 /* 500 * Reserve one segment for kernel virtual memory 501 */ 502 *start = (vaddr_t)(KERNEL_SR << ADDR_SR_SHFT); 503 *end = *start + SEGMENT_LENGTH; 504 #else 505 *start = (vaddr_t) VM_MIN_KERNEL_ADDRESS; 506 *end = (vaddr_t) VM_MAX_KERNEL_ADDRESS; 507 #endif 508 } 509 510 #ifdef PMAP_GROWKERNEL 511 /* 512 * Preallocate kernel page tables to a specified VA. 513 * This simply loops through the first TTE for each 514 * page table from the beginning of the kernel pmap, 515 * reads the entry, and if the result is 516 * zero (either invalid entry or no page table) it stores 517 * a zero there, populating page tables in the process. 518 * This is not the most efficient technique but i don't 519 * expect it to be called that often. 520 */ 521 extern struct vm_page *vm_page_alloc1(void); 522 extern void vm_page_free1(struct vm_page *); 523 524 vaddr_t kbreak = VM_MIN_KERNEL_ADDRESS; 525 526 vaddr_t 527 pmap_growkernel(vaddr_t maxkvaddr) 528 { 529 int s; 530 int seg; 531 paddr_t pg; 532 struct pmap *pm = pmap_kernel(); 533 534 s = splvm(); 535 536 /* Align with the start of a page table */ 537 for (kbreak &= ~(PTMAP-1); kbreak < maxkvaddr; 538 kbreak += PTMAP) { 539 seg = STIDX(kbreak); 540 541 if (pte_find(pm, kbreak)) 542 continue; 543 544 if (uvm.page_init_done) { 545 pg = (paddr_t)VM_PAGE_TO_PHYS(vm_page_alloc1()); 546 } else { 547 if (!uvm_page_physget(&pg)) 548 panic("pmap_growkernel: no memory"); 549 } 550 if (!pg) 551 panic("pmap_growkernel: no pages"); 552 pmap_zero_page((paddr_t)pg); 553 554 /* XXX This is based on all phymem being addressable */ 555 pm->pm_ptbl[seg] = (u_int *)pg; 556 } 557 splx(s); 558 return (kbreak); 559 } 560 561 /* 562 * vm_page_alloc1: 563 * 564 * Allocate and return a memory cell with no associated object. 565 */ 566 struct vm_page * 567 vm_page_alloc1(void) 568 { 569 struct vm_page *pg; 570 571 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE); 572 if (pg) { 573 pg->wire_count = 1; /* no mappings yet */ 574 pg->flags &= ~PG_BUSY; /* never busy */ 575 } 576 return pg; 577 } 578 579 /* 580 * vm_page_free1: 581 * 582 * Returns the given page to the free list, 583 * disassociating it with any VM object. 584 * 585 * Object and page must be locked prior to entry. 586 */ 587 void 588 vm_page_free1(struct vm_page *pg) 589 { 590 #ifdef DIAGNOSTIC 591 if (pg->flags != (PG_CLEAN|PG_FAKE)) { 592 printf("Freeing invalid page %p\n", pg); 593 printf("pa = %llx\n", (unsigned long long)VM_PAGE_TO_PHYS(pg)); 594 #ifdef DDB 595 Debugger(); 596 #endif 597 return; 598 } 599 #endif 600 pg->flags |= PG_BUSY; 601 pg->wire_count = 0; 602 uvm_pagefree(pg); 603 } 604 #endif 605 606 /* 607 * Create and return a physical map. 608 */ 609 struct pmap * 610 pmap_create(void) 611 { 612 struct pmap *pm; 613 614 pm = malloc(sizeof *pm, M_VMPMAP, M_WAITOK); 615 memset(pm, 0, sizeof *pm); 616 pm->pm_refs = 1; 617 return pm; 618 } 619 620 /* 621 * Add a reference to the given pmap. 622 */ 623 void 624 pmap_reference(struct pmap *pm) 625 { 626 627 pm->pm_refs++; 628 } 629 630 /* 631 * Retire the given pmap from service. 632 * Should only be called if the map contains no valid mappings. 633 */ 634 void 635 pmap_destroy(struct pmap *pm) 636 { 637 int i; 638 639 if (--pm->pm_refs > 0) { 640 return; 641 } 642 KASSERT(pm->pm_stats.resident_count == 0); 643 KASSERT(pm->pm_stats.wired_count == 0); 644 for (i = 0; i < STSZ; i++) 645 if (pm->pm_ptbl[i]) { 646 uvm_km_free(kernel_map, (vaddr_t)pm->pm_ptbl[i], 647 PAGE_SIZE, UVM_KMF_WIRED); 648 pm->pm_ptbl[i] = NULL; 649 } 650 if (pm->pm_ctx) 651 ctx_free(pm); 652 free(pm, M_VMPMAP); 653 } 654 655 /* 656 * Copy the range specified by src_addr/len 657 * from the source map to the range dst_addr/len 658 * in the destination map. 659 * 660 * This routine is only advisory and need not do anything. 661 */ 662 void 663 pmap_copy(struct pmap *dst_pmap, struct pmap *src_pmap, vaddr_t dst_addr, 664 vsize_t len, vaddr_t src_addr) 665 { 666 } 667 668 /* 669 * Require that all active physical maps contain no 670 * incorrect entries NOW. 671 */ 672 void 673 pmap_update(struct pmap *pmap) 674 { 675 } 676 677 /* 678 * Garbage collects the physical map system for 679 * pages which are no longer used. 680 * Success need not be guaranteed -- that is, there 681 * may well be pages which are not referenced, but 682 * others may be collected. 683 * Called by the pageout daemon when pages are scarce. 684 */ 685 void 686 pmap_collect(struct pmap *pm) 687 { 688 } 689 690 /* 691 * Fill the given physical page with zeroes. 692 */ 693 void 694 pmap_zero_page(paddr_t pa) 695 { 696 697 #ifdef PPC_4XX_NOCACHE 698 memset((void *)pa, 0, PAGE_SIZE); 699 #else 700 int i; 701 702 for (i = PAGE_SIZE/CACHELINESIZE; i > 0; i--) { 703 __asm volatile ("dcbz 0,%0" :: "r"(pa)); 704 pa += CACHELINESIZE; 705 } 706 #endif 707 } 708 709 /* 710 * Copy the given physical source page to its destination. 711 */ 712 void 713 pmap_copy_page(paddr_t src, paddr_t dst) 714 { 715 716 memcpy((void *)dst, (void *)src, PAGE_SIZE); 717 dcache_flush_page(dst); 718 } 719 720 /* 721 * This returns != 0 on success. 722 */ 723 static inline int 724 pmap_enter_pv(struct pmap *pm, vaddr_t va, paddr_t pa, int flags) 725 { 726 struct pv_entry *pv, *npv = NULL; 727 int s; 728 729 if (!pmap_initialized) 730 return 0; 731 732 s = splvm(); 733 pv = pa_to_pv(pa); 734 if (!pv->pv_pm) { 735 /* 736 * No entries yet, use header as the first entry. 737 */ 738 pv->pv_va = va; 739 pv->pv_pm = pm; 740 pv->pv_next = NULL; 741 } else { 742 /* 743 * There is at least one other VA mapping this page. 744 * Place this entry after the header. 745 */ 746 npv = pool_get(&pv_pool, PR_NOWAIT); 747 if (npv == NULL) { 748 if ((flags & PMAP_CANFAIL) == 0) 749 panic("pmap_enter_pv: failed"); 750 splx(s); 751 return 0; 752 } 753 npv->pv_va = va; 754 npv->pv_pm = pm; 755 npv->pv_next = pv->pv_next; 756 pv->pv_next = npv; 757 pv = npv; 758 } 759 if (flags & PMAP_WIRED) { 760 PV_WIRE(pv); 761 pm->pm_stats.wired_count++; 762 } 763 splx(s); 764 return (1); 765 } 766 767 static void 768 pmap_remove_pv(struct pmap *pm, vaddr_t va, paddr_t pa) 769 { 770 struct pv_entry *pv, *npv; 771 772 /* 773 * Remove from the PV table. 774 */ 775 pv = pa_to_pv(pa); 776 if (!pv) 777 return; 778 779 /* 780 * If it is the first entry on the list, it is actually 781 * in the header and we must copy the following entry up 782 * to the header. Otherwise we must search the list for 783 * the entry. In either case we free the now unused entry. 784 */ 785 if (pm == pv->pv_pm && PV_CMPVA(va, pv)) { 786 if (PV_ISWIRED(pv)) { 787 pm->pm_stats.wired_count--; 788 } 789 if ((npv = pv->pv_next)) { 790 *pv = *npv; 791 pool_put(&pv_pool, npv); 792 } else 793 pv->pv_pm = NULL; 794 } else { 795 for (; (npv = pv->pv_next) != NULL; pv = npv) 796 if (pm == npv->pv_pm && PV_CMPVA(va, npv)) 797 break; 798 if (npv) { 799 pv->pv_next = npv->pv_next; 800 if (PV_ISWIRED(npv)) { 801 pm->pm_stats.wired_count--; 802 } 803 pool_put(&pv_pool, npv); 804 } 805 } 806 } 807 808 /* 809 * Insert physical page at pa into the given pmap at virtual address va. 810 */ 811 int 812 pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags) 813 { 814 int s; 815 u_int tte; 816 int managed; 817 818 /* 819 * Have to remove any existing mapping first. 820 */ 821 pmap_remove(pm, va, va + PAGE_SIZE); 822 823 if (flags & PMAP_WIRED) 824 flags |= prot; 825 826 managed = 0; 827 if (vm_physseg_find(atop(pa), NULL) != -1) 828 managed = 1; 829 830 /* 831 * Generate TTE. 832 */ 833 tte = TTE_PA(pa); 834 /* XXXX -- need to support multiple page sizes. */ 835 tte |= TTE_SZ_16K; 836 #ifdef DIAGNOSTIC 837 if ((flags & (PME_NOCACHE | PME_WRITETHROUG)) == 838 (PME_NOCACHE | PME_WRITETHROUG)) 839 panic("pmap_enter: uncached & writethrough"); 840 #endif 841 if (flags & PME_NOCACHE) 842 /* Must be I/O mapping */ 843 tte |= TTE_I | TTE_G; 844 #ifdef PPC_4XX_NOCACHE 845 tte |= TTE_I; 846 #else 847 else if (flags & PME_WRITETHROUG) 848 /* Uncached and writethrough are not compatible */ 849 tte |= TTE_W; 850 #endif 851 if (pm == pmap_kernel()) 852 tte |= TTE_ZONE(ZONE_PRIV); 853 else 854 tte |= TTE_ZONE(ZONE_USER); 855 856 if (flags & VM_PROT_WRITE) 857 tte |= TTE_WR; 858 859 if (flags & VM_PROT_EXECUTE) 860 tte |= TTE_EX; 861 862 /* 863 * Now record mapping for later back-translation. 864 */ 865 if (pmap_initialized && managed) { 866 char *attr; 867 868 if (!pmap_enter_pv(pm, va, pa, flags)) { 869 /* Could not enter pv on a managed page */ 870 return 1; 871 } 872 873 /* Now set attributes. */ 874 attr = pa_to_attr(pa); 875 #ifdef DIAGNOSTIC 876 if (!attr) 877 panic("managed but no attr"); 878 #endif 879 if (flags & VM_PROT_ALL) 880 *attr |= PMAP_ATTR_REF; 881 if (flags & VM_PROT_WRITE) 882 *attr |= PMAP_ATTR_CHG; 883 } 884 885 s = splvm(); 886 887 /* Insert page into page table. */ 888 pte_enter(pm, va, tte); 889 890 /* If this is a real fault, enter it in the tlb */ 891 if (tte && ((flags & PMAP_WIRED) == 0)) { 892 ppc4xx_tlb_enter(pm->pm_ctx, va, tte); 893 } 894 splx(s); 895 896 /* Flush the real memory from the instruction cache. */ 897 if ((prot & VM_PROT_EXECUTE) && (tte & TTE_I) == 0) 898 __syncicache((void *)pa, PAGE_SIZE); 899 900 return 0; 901 } 902 903 void 904 pmap_unwire(struct pmap *pm, vaddr_t va) 905 { 906 struct pv_entry *pv; 907 paddr_t pa; 908 int s; 909 910 if (!pmap_extract(pm, va, &pa)) { 911 return; 912 } 913 914 pv = pa_to_pv(pa); 915 if (!pv) 916 return; 917 918 s = splvm(); 919 while (pv != NULL) { 920 if (pm == pv->pv_pm && PV_CMPVA(va, pv)) { 921 if (PV_ISWIRED(pv)) { 922 PV_UNWIRE(pv); 923 pm->pm_stats.wired_count--; 924 } 925 break; 926 } 927 pv = pv->pv_next; 928 } 929 splx(s); 930 } 931 932 void 933 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) 934 { 935 int s; 936 u_int tte; 937 struct pmap *pm = pmap_kernel(); 938 939 /* 940 * Have to remove any existing mapping first. 941 */ 942 943 /* 944 * Generate TTE. 945 * 946 * XXXX 947 * 948 * Since the kernel does not handle execution privileges properly, 949 * we will handle read and execute permissions together. 950 */ 951 tte = 0; 952 if (prot & VM_PROT_ALL) { 953 954 tte = TTE_PA(pa) | TTE_EX | TTE_ZONE(ZONE_PRIV); 955 /* XXXX -- need to support multiple page sizes. */ 956 tte |= TTE_SZ_16K; 957 #ifdef DIAGNOSTIC 958 if ((prot & (PME_NOCACHE | PME_WRITETHROUG)) == 959 (PME_NOCACHE | PME_WRITETHROUG)) 960 panic("pmap_kenter_pa: uncached & writethrough"); 961 #endif 962 if (prot & PME_NOCACHE) 963 /* Must be I/O mapping */ 964 tte |= TTE_I | TTE_G; 965 #ifdef PPC_4XX_NOCACHE 966 tte |= TTE_I; 967 #else 968 else if (prot & PME_WRITETHROUG) 969 /* Uncached and writethrough are not compatible */ 970 tte |= TTE_W; 971 #endif 972 if (prot & VM_PROT_WRITE) 973 tte |= TTE_WR; 974 } 975 976 s = splvm(); 977 978 /* Insert page into page table. */ 979 pte_enter(pm, va, tte); 980 splx(s); 981 } 982 983 void 984 pmap_kremove(vaddr_t va, vsize_t len) 985 { 986 987 while (len > 0) { 988 pte_enter(pmap_kernel(), va, 0); 989 va += PAGE_SIZE; 990 len -= PAGE_SIZE; 991 } 992 } 993 994 /* 995 * Remove the given range of mapping entries. 996 */ 997 void 998 pmap_remove(struct pmap *pm, vaddr_t va, vaddr_t endva) 999 { 1000 int s; 1001 paddr_t pa; 1002 volatile u_int *ptp; 1003 1004 s = splvm(); 1005 while (va < endva) { 1006 1007 if ((ptp = pte_find(pm, va)) && (pa = *ptp)) { 1008 pa = TTE_PA(pa); 1009 pmap_remove_pv(pm, va, pa); 1010 *ptp = 0; 1011 ppc4xx_tlb_flush(va, pm->pm_ctx); 1012 pm->pm_stats.resident_count--; 1013 } 1014 va += PAGE_SIZE; 1015 } 1016 1017 splx(s); 1018 } 1019 1020 /* 1021 * Get the physical page address for the given pmap/virtual address. 1022 */ 1023 bool 1024 pmap_extract(struct pmap *pm, vaddr_t va, paddr_t *pap) 1025 { 1026 int seg = STIDX(va); 1027 int ptn = PTIDX(va); 1028 u_int pa = 0; 1029 int s; 1030 1031 s = splvm(); 1032 if (pm->pm_ptbl[seg] && (pa = pm->pm_ptbl[seg][ptn])) { 1033 *pap = TTE_PA(pa) | (va & PGOFSET); 1034 } 1035 splx(s); 1036 return (pa != 0); 1037 } 1038 1039 /* 1040 * Lower the protection on the specified range of this pmap. 1041 * 1042 * There are only two cases: either the protection is going to 0, 1043 * or it is going to read-only. 1044 */ 1045 void 1046 pmap_protect(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1047 { 1048 volatile u_int *ptp; 1049 int s, bic; 1050 1051 if ((prot & VM_PROT_READ) == 0) { 1052 pmap_remove(pm, sva, eva); 1053 return; 1054 } 1055 bic = 0; 1056 if ((prot & VM_PROT_WRITE) == 0) { 1057 bic |= TTE_WR; 1058 } 1059 if ((prot & VM_PROT_EXECUTE) == 0) { 1060 bic |= TTE_EX; 1061 } 1062 if (bic == 0) { 1063 return; 1064 } 1065 s = splvm(); 1066 while (sva < eva) { 1067 if ((ptp = pte_find(pm, sva)) != NULL) { 1068 *ptp &= ~bic; 1069 ppc4xx_tlb_flush(sva, pm->pm_ctx); 1070 } 1071 sva += PAGE_SIZE; 1072 } 1073 splx(s); 1074 } 1075 1076 bool 1077 pmap_check_attr(struct vm_page *pg, u_int mask, int clear) 1078 { 1079 paddr_t pa; 1080 char *attr; 1081 int s, rv; 1082 1083 /* 1084 * First modify bits in cache. 1085 */ 1086 pa = VM_PAGE_TO_PHYS(pg); 1087 attr = pa_to_attr(pa); 1088 if (attr == NULL) 1089 return false; 1090 1091 s = splvm(); 1092 rv = ((*attr & mask) != 0); 1093 if (clear) { 1094 *attr &= ~mask; 1095 pmap_page_protect(pg, mask == PMAP_ATTR_CHG ? VM_PROT_READ : 0); 1096 } 1097 splx(s); 1098 return rv; 1099 } 1100 1101 1102 /* 1103 * Lower the protection on the specified physical page. 1104 * 1105 * There are only two cases: either the protection is going to 0, 1106 * or it is going to read-only. 1107 */ 1108 void 1109 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 1110 { 1111 paddr_t pa = VM_PAGE_TO_PHYS(pg); 1112 vaddr_t va; 1113 struct pv_entry *pvh, *pv, *npv; 1114 struct pmap *pm; 1115 1116 pvh = pa_to_pv(pa); 1117 if (pvh == NULL) 1118 return; 1119 1120 /* Handle extra pvs which may be deleted in the operation */ 1121 for (pv = pvh->pv_next; pv; pv = npv) { 1122 npv = pv->pv_next; 1123 1124 pm = pv->pv_pm; 1125 va = pv->pv_va; 1126 pmap_protect(pm, va, va + PAGE_SIZE, prot); 1127 } 1128 /* Now check the head pv */ 1129 if (pvh->pv_pm) { 1130 pv = pvh; 1131 pm = pv->pv_pm; 1132 va = pv->pv_va; 1133 pmap_protect(pm, va, va + PAGE_SIZE, prot); 1134 } 1135 } 1136 1137 /* 1138 * Activate the address space for the specified process. If the process 1139 * is the current process, load the new MMU context. 1140 */ 1141 void 1142 pmap_activate(struct lwp *l) 1143 { 1144 #if 0 1145 struct pcb *pcb = &l->l_proc->p_addr->u_pcb; 1146 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 1147 1148 /* 1149 * XXX Normally performed in cpu_fork(). 1150 */ 1151 printf("pmap_activate(%p), pmap=%p\n",l,pmap); 1152 pcb->pcb_pm = pmap; 1153 #endif 1154 } 1155 1156 /* 1157 * Deactivate the specified process's address space. 1158 */ 1159 void 1160 pmap_deactivate(struct lwp *l) 1161 { 1162 } 1163 1164 /* 1165 * Synchronize caches corresponding to [addr, addr+len) in p. 1166 */ 1167 void 1168 pmap_procwr(struct proc *p, vaddr_t va, size_t len) 1169 { 1170 struct pmap *pm = p->p_vmspace->vm_map.pmap; 1171 int msr, ctx, opid, step; 1172 1173 step = CACHELINESIZE; 1174 1175 /* 1176 * Need to turn off IMMU and switch to user context. 1177 * (icbi uses DMMU). 1178 */ 1179 if (!(ctx = pm->pm_ctx)) { 1180 /* No context -- assign it one */ 1181 ctx_alloc(pm); 1182 ctx = pm->pm_ctx; 1183 } 1184 __asm volatile("mfmsr %0;" 1185 "li %1, %7;" 1186 "andc %1,%0,%1;" 1187 "mtmsr %1;" 1188 "sync;isync;" 1189 "mfpid %1;" 1190 "mtpid %2;" 1191 "sync; isync;" 1192 "1:" 1193 "dcbf 0,%3;" 1194 "icbi 0,%3;" 1195 "add %3,%3,%5;" 1196 "addc. %4,%4,%6;" 1197 "bge 1b;" 1198 "mtpid %1;" 1199 "mtmsr %0;" 1200 "sync; isync" 1201 : "=&r" (msr), "=&r" (opid) 1202 : "r" (ctx), "r" (va), "r" (len), "r" (step), "r" (-step), 1203 "K" (PSL_IR | PSL_DR)); 1204 } 1205 1206 1207 /* This has to be done in real mode !!! */ 1208 void 1209 ppc4xx_tlb_flush(vaddr_t va, int pid) 1210 { 1211 u_long i, found; 1212 u_long msr; 1213 1214 /* If there's no context then it can't be mapped. */ 1215 if (!pid) 1216 return; 1217 1218 __asm( "mfpid %1;" /* Save PID */ 1219 "mfmsr %2;" /* Save MSR */ 1220 "li %0,0;" /* Now clear MSR */ 1221 "mtmsr %0;" 1222 "mtpid %4;" /* Set PID */ 1223 "sync;" 1224 "tlbsx. %0,0,%3;" /* Search TLB */ 1225 "sync;" 1226 "mtpid %1;" /* Restore PID */ 1227 "mtmsr %2;" /* Restore MSR */ 1228 "sync;isync;" 1229 "li %1,1;" 1230 "beq 1f;" 1231 "li %1,0;" 1232 "1:" 1233 : "=&r" (i), "=&r" (found), "=&r" (msr) 1234 : "r" (va), "r" (pid)); 1235 if (found && !TLB_LOCKED(i)) { 1236 1237 /* Now flush translation */ 1238 __asm volatile( 1239 "tlbwe %0,%1,0;" 1240 "sync;isync;" 1241 : : "r" (0), "r" (i)); 1242 1243 tlb_info[i].ti_ctx = 0; 1244 tlb_info[i].ti_flags = 0; 1245 tlbnext = i; 1246 /* Successful flushes */ 1247 tlbflush_ev.ev_count++; 1248 } 1249 } 1250 1251 void 1252 ppc4xx_tlb_flush_all(void) 1253 { 1254 u_long i; 1255 1256 for (i = 0; i < NTLB; i++) 1257 if (!TLB_LOCKED(i)) { 1258 __asm volatile( 1259 "tlbwe %0,%1,0;" 1260 "sync;isync;" 1261 : : "r" (0), "r" (i)); 1262 tlb_info[i].ti_ctx = 0; 1263 tlb_info[i].ti_flags = 0; 1264 } 1265 1266 __asm volatile("sync;isync"); 1267 } 1268 1269 /* Find a TLB entry to evict. */ 1270 static int 1271 ppc4xx_tlb_find_victim(void) 1272 { 1273 int flags; 1274 1275 for (;;) { 1276 if (++tlbnext >= NTLB) 1277 tlbnext = tlb_nreserved; 1278 flags = tlb_info[tlbnext].ti_flags; 1279 if (!(flags & TLBF_USED) || 1280 (flags & (TLBF_LOCKED | TLBF_REF)) == 0) { 1281 u_long va, stack = (u_long)&va; 1282 1283 if (!((tlb_info[tlbnext].ti_va ^ stack) & (~PGOFSET)) && 1284 (tlb_info[tlbnext].ti_ctx == KERNEL_PID) && 1285 (flags & TLBF_USED)) { 1286 /* Kernel stack page */ 1287 flags |= TLBF_USED; 1288 tlb_info[tlbnext].ti_flags = flags; 1289 } else { 1290 /* Found it! */ 1291 return (tlbnext); 1292 } 1293 } else { 1294 tlb_info[tlbnext].ti_flags = (flags & ~TLBF_REF); 1295 } 1296 } 1297 } 1298 1299 void 1300 ppc4xx_tlb_enter(int ctx, vaddr_t va, u_int pte) 1301 { 1302 u_long th, tl, idx; 1303 tlbpid_t pid; 1304 u_short msr; 1305 paddr_t pa; 1306 int s, sz; 1307 1308 tlbenter_ev.ev_count++; 1309 1310 sz = (pte & TTE_SZ_MASK) >> TTE_SZ_SHIFT; 1311 pa = (pte & TTE_RPN_MASK(sz)); 1312 th = (va & TLB_EPN_MASK) | (sz << TLB_SIZE_SHFT) | TLB_VALID; 1313 tl = (pte & ~TLB_RPN_MASK) | pa; 1314 tl |= ppc4xx_tlbflags(va, pa); 1315 1316 s = splhigh(); 1317 idx = ppc4xx_tlb_find_victim(); 1318 1319 #ifdef DIAGNOSTIC 1320 if ((idx < tlb_nreserved) || (idx >= NTLB)) { 1321 panic("ppc4xx_tlb_enter: replacing entry %ld", idx); 1322 } 1323 #endif 1324 1325 tlb_info[idx].ti_va = (va & TLB_EPN_MASK); 1326 tlb_info[idx].ti_ctx = ctx; 1327 tlb_info[idx].ti_flags = TLBF_USED | TLBF_REF; 1328 1329 __asm volatile( 1330 "mfmsr %0;" /* Save MSR */ 1331 "li %1,0;" 1332 "tlbwe %1,%3,0;" /* Invalidate old entry. */ 1333 "mtmsr %1;" /* Clear MSR */ 1334 "mfpid %1;" /* Save old PID */ 1335 "mtpid %2;" /* Load translation ctx */ 1336 "sync; isync;" 1337 #ifdef DEBUG 1338 "andi. %3,%3,63;" 1339 "tweqi %3,0;" /* XXXXX DEBUG trap on index 0 */ 1340 #endif 1341 "tlbwe %4,%3,1; tlbwe %5,%3,0;" /* Set TLB */ 1342 "sync; isync;" 1343 "mtpid %1; mtmsr %0;" /* Restore PID and MSR */ 1344 "sync; isync;" 1345 : "=&r" (msr), "=&r" (pid) 1346 : "r" (ctx), "r" (idx), "r" (tl), "r" (th)); 1347 splx(s); 1348 } 1349 1350 void 1351 ppc4xx_tlb_init(void) 1352 { 1353 int i; 1354 1355 /* Mark reserved TLB entries */ 1356 for (i = 0; i < tlb_nreserved; i++) { 1357 tlb_info[i].ti_flags = TLBF_LOCKED | TLBF_USED; 1358 tlb_info[i].ti_ctx = KERNEL_PID; 1359 } 1360 1361 /* Setup security zones */ 1362 /* Z0 - accessible by kernel only if TLB entry permissions allow 1363 * Z1,Z2 - access is controlled by TLB entry permissions 1364 * Z3 - full access regardless of TLB entry permissions 1365 */ 1366 1367 __asm volatile( 1368 "mtspr %0,%1;" 1369 "sync;" 1370 :: "K"(SPR_ZPR), "r" (0x1b000000)); 1371 } 1372 1373 /* 1374 * ppc4xx_tlb_size_mask: 1375 * 1376 * Roundup size to supported page size, return TLBHI mask and real size. 1377 */ 1378 static int 1379 ppc4xx_tlb_size_mask(size_t size, int *mask, int *rsiz) 1380 { 1381 int i; 1382 1383 for (i = 0; i < __arraycount(tlbsize); i++) 1384 if (size <= tlbsize[i]) { 1385 *mask = (i << TLB_SIZE_SHFT); 1386 *rsiz = tlbsize[i]; 1387 return (0); 1388 } 1389 return (EINVAL); 1390 } 1391 1392 /* 1393 * ppc4xx_tlb_mapiodev: 1394 * 1395 * Lookup virtual address of mapping previously entered via 1396 * ppc4xx_tlb_reserve. Search TLB directly so that we don't 1397 * need to waste extra storage for reserved mappings. Note 1398 * that reading TLBHI also sets PID, but all reserved mappings 1399 * use KERNEL_PID, so the side effect is nil. 1400 */ 1401 void * 1402 ppc4xx_tlb_mapiodev(paddr_t base, psize_t len) 1403 { 1404 paddr_t pa; 1405 vaddr_t va; 1406 u_int lo, hi, sz; 1407 int i; 1408 1409 /* tlb_nreserved is only allowed to grow, so this is safe. */ 1410 for (i = 0; i < tlb_nreserved; i++) { 1411 __asm volatile ( 1412 " tlbre %0,%2,1 \n" /* TLBLO */ 1413 " tlbre %1,%2,0 \n" /* TLBHI */ 1414 : "=&r" (lo), "=&r" (hi) 1415 : "r" (i)); 1416 1417 KASSERT(hi & TLB_VALID); 1418 KASSERT(mfspr(SPR_PID) == KERNEL_PID); 1419 1420 pa = (lo & TLB_RPN_MASK); 1421 if (base < pa) 1422 continue; 1423 1424 sz = tlbsize[(hi & TLB_SIZE_MASK) >> TLB_SIZE_SHFT]; 1425 if ((base + len) > (pa + sz)) 1426 continue; 1427 1428 va = (hi & TLB_EPN_MASK) + (base & (sz - 1)); /* sz = 2^n */ 1429 return (void *)(va); 1430 } 1431 1432 return (NULL); 1433 } 1434 1435 /* 1436 * ppc4xx_tlb_reserve: 1437 * 1438 * Map physical range to kernel virtual chunk via reserved TLB entry. 1439 */ 1440 void 1441 ppc4xx_tlb_reserve(paddr_t pa, vaddr_t va, size_t size, int flags) 1442 { 1443 u_int lo, hi; 1444 int szmask, rsize; 1445 1446 /* Called before pmap_bootstrap(), va outside kernel space. */ 1447 KASSERT(va < VM_MIN_KERNEL_ADDRESS || va >= VM_MAX_KERNEL_ADDRESS); 1448 KASSERT(! pmap_bootstrap_done); 1449 KASSERT(tlb_nreserved < NTLB); 1450 1451 /* Resolve size. */ 1452 if (ppc4xx_tlb_size_mask(size, &szmask, &rsize) != 0) 1453 panic("ppc4xx_tlb_reserve: entry %d, %zuB too large", 1454 size, tlb_nreserved); 1455 1456 /* Real size will be power of two >= 1024, so this is OK. */ 1457 pa &= ~(rsize - 1); /* RPN */ 1458 va &= ~(rsize - 1); /* EPN */ 1459 1460 lo = pa | TLB_WR | flags; 1461 hi = va | TLB_VALID | szmask; 1462 1463 #ifdef PPC_4XX_NOCACHE 1464 lo |= TLB_I; 1465 #endif 1466 1467 __asm volatile( 1468 " tlbwe %1,%0,1 \n" /* write TLBLO */ 1469 " tlbwe %2,%0,0 \n" /* write TLBHI */ 1470 " sync \n" 1471 " isync \n" 1472 : : "r" (tlb_nreserved), "r" (lo), "r" (hi)); 1473 1474 tlb_nreserved++; 1475 } 1476 1477 /* 1478 * We should pass the ctx in from trap code. 1479 */ 1480 int 1481 pmap_tlbmiss(vaddr_t va, int ctx) 1482 { 1483 volatile u_int *pte; 1484 u_long tte; 1485 1486 tlbmiss_ev.ev_count++; 1487 1488 /* 1489 * We will reserve 0 upto VM_MIN_KERNEL_ADDRESS for va == pa mappings. 1490 * Physical RAM is expected to live in this range, care must be taken 1491 * to not clobber 0 upto ${physmem} with device mappings in machdep 1492 * code. 1493 */ 1494 if (ctx != KERNEL_PID || va >= VM_MIN_KERNEL_ADDRESS) { 1495 pte = pte_find((struct pmap *)__UNVOLATILE(ctxbusy[ctx]), va); 1496 if (pte == NULL) { 1497 /* Map unmanaged addresses directly for kernel access */ 1498 return 1; 1499 } 1500 tte = *pte; 1501 if (tte == 0) { 1502 return 1; 1503 } 1504 } else { 1505 /* Create a 16MB writable mapping. */ 1506 #ifdef PPC_4XX_NOCACHE 1507 tte = TTE_PA(va) | TTE_ZONE(ZONE_PRIV) | TTE_SZ_16M | TTE_I |TTE_WR; 1508 #else 1509 tte = TTE_PA(va) | TTE_ZONE(ZONE_PRIV) | TTE_SZ_16M | TTE_WR; 1510 #endif 1511 } 1512 tlbhit_ev.ev_count++; 1513 ppc4xx_tlb_enter(ctx, va, tte); 1514 1515 return 0; 1516 } 1517 1518 /* 1519 * Flush all the entries matching a context from the TLB. 1520 */ 1521 static int 1522 ctx_flush(int cnum) 1523 { 1524 int i; 1525 1526 /* We gotta steal this context */ 1527 for (i = tlb_nreserved; i < NTLB; i++) { 1528 if (tlb_info[i].ti_ctx == cnum) { 1529 /* Can't steal ctx if it has a locked entry. */ 1530 if (TLB_LOCKED(i)) { 1531 #ifdef DIAGNOSTIC 1532 printf("ctx_flush: can't invalidate " 1533 "locked mapping %d " 1534 "for context %d\n", i, cnum); 1535 #ifdef DDB 1536 Debugger(); 1537 #endif 1538 #endif 1539 return (1); 1540 } 1541 #ifdef DIAGNOSTIC 1542 if (i < tlb_nreserved) 1543 panic("TLB entry %d not locked", i); 1544 #endif 1545 /* Invalidate particular TLB entry regardless of locked status */ 1546 __asm volatile("tlbwe %0,%1,0" : :"r"(0),"r"(i)); 1547 tlb_info[i].ti_flags = 0; 1548 } 1549 } 1550 return (0); 1551 } 1552 1553 /* 1554 * Allocate a context. If necessary, steal one from someone else. 1555 * 1556 * The new context is flushed from the TLB before returning. 1557 */ 1558 int 1559 ctx_alloc(struct pmap *pm) 1560 { 1561 int s, cnum; 1562 static int next = MINCTX; 1563 1564 if (pm == pmap_kernel()) { 1565 #ifdef DIAGNOSTIC 1566 printf("ctx_alloc: kernel pmap!\n"); 1567 #endif 1568 return (0); 1569 } 1570 s = splvm(); 1571 1572 /* Find a likely context. */ 1573 cnum = next; 1574 do { 1575 if ((++cnum) > NUMCTX) 1576 cnum = MINCTX; 1577 } while (ctxbusy[cnum] != NULL && cnum != next); 1578 1579 /* Now clean it out */ 1580 oops: 1581 if (cnum < MINCTX) 1582 cnum = MINCTX; /* Never steal ctx 0 or 1 */ 1583 if (ctx_flush(cnum)) { 1584 /* oops -- something's wired. */ 1585 if ((++cnum) > NUMCTX) 1586 cnum = MINCTX; 1587 goto oops; 1588 } 1589 1590 if (ctxbusy[cnum]) { 1591 #ifdef DEBUG 1592 /* We should identify this pmap and clear it */ 1593 printf("Warning: stealing context %d\n", cnum); 1594 #endif 1595 ctxbusy[cnum]->pm_ctx = 0; 1596 } 1597 ctxbusy[cnum] = pm; 1598 next = cnum; 1599 splx(s); 1600 pm->pm_ctx = cnum; 1601 1602 return cnum; 1603 } 1604 1605 /* 1606 * Give away a context. 1607 */ 1608 void 1609 ctx_free(struct pmap *pm) 1610 { 1611 int oldctx; 1612 1613 oldctx = pm->pm_ctx; 1614 1615 if (oldctx == 0) 1616 panic("ctx_free: freeing kernel context"); 1617 #ifdef DIAGNOSTIC 1618 if (ctxbusy[oldctx] == 0) 1619 printf("ctx_free: freeing free context %d\n", oldctx); 1620 if (ctxbusy[oldctx] != pm) { 1621 printf("ctx_free: freeing someone esle's context\n " 1622 "ctxbusy[%d] = %p, pm->pm_ctx = %p\n", 1623 oldctx, (void *)(u_long)ctxbusy[oldctx], pm); 1624 #ifdef DDB 1625 Debugger(); 1626 #endif 1627 } 1628 #endif 1629 /* We should verify it has not been stolen and reallocated... */ 1630 ctxbusy[oldctx] = NULL; 1631 ctx_flush(oldctx); 1632 } 1633 1634 1635 #ifdef DEBUG 1636 /* 1637 * Test ref/modify handling. 1638 */ 1639 void pmap_testout(void); 1640 void 1641 pmap_testout(void) 1642 { 1643 vaddr_t va; 1644 volatile int *loc; 1645 int val = 0; 1646 paddr_t pa; 1647 struct vm_page *pg; 1648 int ref, mod; 1649 1650 /* Allocate a page */ 1651 va = (vaddr_t)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 1652 UVM_KMF_WIRED | UVM_KMF_ZERO); 1653 loc = (int*)va; 1654 1655 pmap_extract(pmap_kernel(), va, &pa); 1656 pg = PHYS_TO_VM_PAGE(pa); 1657 pmap_unwire(pmap_kernel(), va); 1658 1659 pmap_kremove(va, PAGE_SIZE); 1660 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1661 pmap_update(pmap_kernel()); 1662 1663 /* Now clear reference and modify */ 1664 ref = pmap_clear_reference(pg); 1665 mod = pmap_clear_modify(pg); 1666 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1667 (void *)(u_long)va, (long)pa, 1668 ref, mod); 1669 1670 /* Check it's properly cleared */ 1671 ref = pmap_is_referenced(pg); 1672 mod = pmap_is_modified(pg); 1673 printf("Checking cleared page: ref %d, mod %d\n", 1674 ref, mod); 1675 1676 /* Reference page */ 1677 val = *loc; 1678 1679 ref = pmap_is_referenced(pg); 1680 mod = pmap_is_modified(pg); 1681 printf("Referenced page: ref %d, mod %d val %x\n", 1682 ref, mod, val); 1683 1684 /* Now clear reference and modify */ 1685 ref = pmap_clear_reference(pg); 1686 mod = pmap_clear_modify(pg); 1687 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1688 (void *)(u_long)va, (long)pa, 1689 ref, mod); 1690 1691 /* Modify page */ 1692 *loc = 1; 1693 1694 ref = pmap_is_referenced(pg); 1695 mod = pmap_is_modified(pg); 1696 printf("Modified page: ref %d, mod %d\n", 1697 ref, mod); 1698 1699 /* Now clear reference and modify */ 1700 ref = pmap_clear_reference(pg); 1701 mod = pmap_clear_modify(pg); 1702 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1703 (void *)(u_long)va, (long)pa, 1704 ref, mod); 1705 1706 /* Check it's properly cleared */ 1707 ref = pmap_is_referenced(pg); 1708 mod = pmap_is_modified(pg); 1709 printf("Checking cleared page: ref %d, mod %d\n", 1710 ref, mod); 1711 1712 /* Modify page */ 1713 *loc = 1; 1714 1715 ref = pmap_is_referenced(pg); 1716 mod = pmap_is_modified(pg); 1717 printf("Modified page: ref %d, mod %d\n", 1718 ref, mod); 1719 1720 /* Check pmap_protect() */ 1721 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_READ); 1722 pmap_update(pmap_kernel()); 1723 ref = pmap_is_referenced(pg); 1724 mod = pmap_is_modified(pg); 1725 printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n", 1726 ref, mod); 1727 1728 /* Now clear reference and modify */ 1729 ref = pmap_clear_reference(pg); 1730 mod = pmap_clear_modify(pg); 1731 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1732 (void *)(u_long)va, (long)pa, 1733 ref, mod); 1734 1735 /* Reference page */ 1736 val = *loc; 1737 1738 ref = pmap_is_referenced(pg); 1739 mod = pmap_is_modified(pg); 1740 printf("Referenced page: ref %d, mod %d val %x\n", 1741 ref, mod, val); 1742 1743 /* Now clear reference and modify */ 1744 ref = pmap_clear_reference(pg); 1745 mod = pmap_clear_modify(pg); 1746 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1747 (void *)(u_long)va, (long)pa, 1748 ref, mod); 1749 1750 /* Modify page */ 1751 #if 0 1752 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1753 pmap_update(pmap_kernel()); 1754 #endif 1755 *loc = 1; 1756 1757 ref = pmap_is_referenced(pg); 1758 mod = pmap_is_modified(pg); 1759 printf("Modified page: ref %d, mod %d\n", 1760 ref, mod); 1761 1762 /* Check pmap_protect() */ 1763 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_NONE); 1764 pmap_update(pmap_kernel()); 1765 ref = pmap_is_referenced(pg); 1766 mod = pmap_is_modified(pg); 1767 printf("pmap_protect(): ref %d, mod %d\n", 1768 ref, mod); 1769 1770 /* Now clear reference and modify */ 1771 ref = pmap_clear_reference(pg); 1772 mod = pmap_clear_modify(pg); 1773 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1774 (void *)(u_long)va, (long)pa, 1775 ref, mod); 1776 1777 /* Reference page */ 1778 val = *loc; 1779 1780 ref = pmap_is_referenced(pg); 1781 mod = pmap_is_modified(pg); 1782 printf("Referenced page: ref %d, mod %d val %x\n", 1783 ref, mod, val); 1784 1785 /* Now clear reference and modify */ 1786 ref = pmap_clear_reference(pg); 1787 mod = pmap_clear_modify(pg); 1788 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1789 (void *)(u_long)va, (long)pa, 1790 ref, mod); 1791 1792 /* Modify page */ 1793 #if 0 1794 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1795 pmap_update(pmap_kernel()); 1796 #endif 1797 *loc = 1; 1798 1799 ref = pmap_is_referenced(pg); 1800 mod = pmap_is_modified(pg); 1801 printf("Modified page: ref %d, mod %d\n", 1802 ref, mod); 1803 1804 /* Check pmap_pag_protect() */ 1805 pmap_page_protect(pg, VM_PROT_READ); 1806 ref = pmap_is_referenced(pg); 1807 mod = pmap_is_modified(pg); 1808 printf("pmap_page_protect(VM_PROT_READ): ref %d, mod %d\n", 1809 ref, mod); 1810 1811 /* Now clear reference and modify */ 1812 ref = pmap_clear_reference(pg); 1813 mod = pmap_clear_modify(pg); 1814 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1815 (void *)(u_long)va, (long)pa, 1816 ref, mod); 1817 1818 /* Reference page */ 1819 val = *loc; 1820 1821 ref = pmap_is_referenced(pg); 1822 mod = pmap_is_modified(pg); 1823 printf("Referenced page: ref %d, mod %d val %x\n", 1824 ref, mod, val); 1825 1826 /* Now clear reference and modify */ 1827 ref = pmap_clear_reference(pg); 1828 mod = pmap_clear_modify(pg); 1829 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1830 (void *)(u_long)va, (long)pa, 1831 ref, mod); 1832 1833 /* Modify page */ 1834 #if 0 1835 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1836 pmap_update(pmap_kernel()); 1837 #endif 1838 *loc = 1; 1839 1840 ref = pmap_is_referenced(pg); 1841 mod = pmap_is_modified(pg); 1842 printf("Modified page: ref %d, mod %d\n", 1843 ref, mod); 1844 1845 /* Check pmap_pag_protect() */ 1846 pmap_page_protect(pg, VM_PROT_NONE); 1847 ref = pmap_is_referenced(pg); 1848 mod = pmap_is_modified(pg); 1849 printf("pmap_page_protect(): ref %d, mod %d\n", 1850 ref, mod); 1851 1852 /* Now clear reference and modify */ 1853 ref = pmap_clear_reference(pg); 1854 mod = pmap_clear_modify(pg); 1855 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1856 (void *)(u_long)va, (long)pa, 1857 ref, mod); 1858 1859 1860 /* Reference page */ 1861 val = *loc; 1862 1863 ref = pmap_is_referenced(pg); 1864 mod = pmap_is_modified(pg); 1865 printf("Referenced page: ref %d, mod %d val %x\n", 1866 ref, mod, val); 1867 1868 /* Now clear reference and modify */ 1869 ref = pmap_clear_reference(pg); 1870 mod = pmap_clear_modify(pg); 1871 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1872 (void *)(u_long)va, (long)pa, 1873 ref, mod); 1874 1875 /* Modify page */ 1876 #if 0 1877 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1878 pmap_update(pmap_kernel()); 1879 #endif 1880 *loc = 1; 1881 1882 ref = pmap_is_referenced(pg); 1883 mod = pmap_is_modified(pg); 1884 printf("Modified page: ref %d, mod %d\n", 1885 ref, mod); 1886 1887 /* Unmap page */ 1888 pmap_remove(pmap_kernel(), va, va+1); 1889 pmap_update(pmap_kernel()); 1890 ref = pmap_is_referenced(pg); 1891 mod = pmap_is_modified(pg); 1892 printf("Unmapped page: ref %d, mod %d\n", ref, mod); 1893 1894 /* Now clear reference and modify */ 1895 ref = pmap_clear_reference(pg); 1896 mod = pmap_clear_modify(pg); 1897 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1898 (void *)(u_long)va, (long)pa, ref, mod); 1899 1900 /* Check it's properly cleared */ 1901 ref = pmap_is_referenced(pg); 1902 mod = pmap_is_modified(pg); 1903 printf("Checking cleared page: ref %d, mod %d\n", 1904 ref, mod); 1905 1906 pmap_remove(pmap_kernel(), va, va + PAGE_SIZE); 1907 pmap_kenter_pa(va, pa, VM_PROT_ALL); 1908 uvm_km_free(kernel_map, (vaddr_t)va, PAGE_SIZE, UVM_KMF_WIRED); 1909 } 1910 #endif 1911