1 /* $NetBSD: pmap.c,v 1.98 2021/04/15 00:00:46 rin Exp $ */ 2 3 /* 4 * Copyright 2001 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Eduardo Horvath and Simon Burge for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 40 * Copyright (C) 1995, 1996 TooLs GmbH. 41 * All rights reserved. 42 * 43 * Redistribution and use in source and binary forms, with or without 44 * modification, are permitted provided that the following conditions 45 * are met: 46 * 1. Redistributions of source code must retain the above copyright 47 * notice, this list of conditions and the following disclaimer. 48 * 2. Redistributions in binary form must reproduce the above copyright 49 * notice, this list of conditions and the following disclaimer in the 50 * documentation and/or other materials provided with the distribution. 51 * 3. All advertising materials mentioning features or use of this software 52 * must display the following acknowledgement: 53 * This product includes software developed by TooLs GmbH. 54 * 4. The name of TooLs GmbH may not be used to endorse or promote products 55 * derived from this software without specific prior written permission. 56 * 57 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 58 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 59 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 60 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 61 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 62 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 63 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 64 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 65 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 66 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 67 */ 68 69 #include <sys/cdefs.h> 70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.98 2021/04/15 00:00:46 rin Exp $"); 71 72 #ifdef _KERNEL_OPT 73 #include "opt_ddb.h" 74 #include "opt_pmap.h" 75 #endif 76 77 #include <sys/param.h> 78 #include <sys/cpu.h> 79 #include <sys/device.h> 80 #include <sys/kmem.h> 81 #include <sys/pool.h> 82 #include <sys/proc.h> 83 #include <sys/queue.h> 84 #include <sys/systm.h> 85 86 #include <uvm/uvm.h> 87 88 #include <machine/powerpc.h> 89 90 #include <powerpc/pcb.h> 91 92 #include <powerpc/spr.h> 93 #include <powerpc/ibm4xx/spr.h> 94 95 #include <powerpc/ibm4xx/cpu.h> 96 #include <powerpc/ibm4xx/tlb.h> 97 98 /* 99 * kernmap is an array of PTEs large enough to map in 100 * 4GB. At 16KB/page it is 256K entries or 2MB. 101 */ 102 #define KERNMAP_SIZE ((0xffffffffU/PAGE_SIZE)+1) 103 void *kernmap; 104 105 #define MINCTX 2 106 #define NUMCTX 256 107 108 volatile struct pmap *ctxbusy[NUMCTX]; 109 110 #define TLBF_USED 0x1 111 #define TLBF_REF 0x2 112 #define TLBF_LOCKED 0x4 113 #define TLB_LOCKED(i) (tlb_info[(i)].ti_flags & TLBF_LOCKED) 114 115 typedef struct tlb_info_s { 116 char ti_flags; 117 char ti_ctx; /* TLB_PID assiciated with the entry */ 118 u_int ti_va; 119 } tlb_info_t; 120 121 volatile tlb_info_t tlb_info[NTLB]; 122 /* We'll use a modified FIFO replacement policy cause it's cheap */ 123 volatile int tlbnext; 124 125 static int tlb_nreserved = 0; 126 static int pmap_bootstrap_done = 0; 127 128 /* Event counters */ 129 struct evcnt tlbmiss_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 130 NULL, "cpu", "tlbmiss"); 131 struct evcnt tlbflush_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 132 NULL, "cpu", "tlbflush"); 133 struct evcnt tlbenter_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP, 134 NULL, "cpu", "tlbenter"); 135 EVCNT_ATTACH_STATIC(tlbmiss_ev); 136 EVCNT_ATTACH_STATIC(tlbflush_ev); 137 EVCNT_ATTACH_STATIC(tlbenter_ev); 138 139 struct pmap kernel_pmap_; 140 struct pmap *const kernel_pmap_ptr = &kernel_pmap_; 141 142 static int npgs; 143 static u_int nextavail; 144 #ifndef MSGBUFADDR 145 extern paddr_t msgbuf_paddr; 146 #endif 147 148 static struct mem_region *mem, *avail; 149 150 /* 151 * This is a cache of referenced/modified bits. 152 * Bits herein are shifted by ATTRSHFT. 153 */ 154 static char *pmap_attrib; 155 156 #define PV_WIRED 0x1 157 #define PV_WIRE(pv) ((pv)->pv_va |= PV_WIRED) 158 #define PV_UNWIRE(pv) ((pv)->pv_va &= ~PV_WIRED) 159 #define PV_ISWIRED(pv) ((pv)->pv_va & PV_WIRED) 160 #define PV_VA(pv) ((pv)->pv_va & ~PV_WIRED) 161 #define PV_CMPVA(va,pv) (!(PV_VA(pv) ^ (va))) 162 163 struct pv_entry { 164 struct pv_entry *pv_next; /* Linked list of mappings */ 165 struct pmap *pv_pm; 166 vaddr_t pv_va; /* virtual address of mapping */ 167 }; 168 169 /* Each index corresponds to TLB_SIZE_* value. */ 170 static size_t tlbsize[] = { 171 1024, /* TLB_SIZE_1K */ 172 4096, /* TLB_SIZE_4K */ 173 16384, /* TLB_SIZE_16K */ 174 65536, /* TLB_SIZE_64K */ 175 262144, /* TLB_SIZE_256K */ 176 1048576, /* TLB_SIZE_1M */ 177 4194304, /* TLB_SIZE_4M */ 178 16777216, /* TLB_SIZE_16M */ 179 }; 180 181 struct pv_entry *pv_table; 182 static struct pool pv_pool; 183 184 static int pmap_initialized; 185 186 static int ctx_flush(int); 187 188 struct pv_entry *pa_to_pv(paddr_t); 189 static inline char *pa_to_attr(paddr_t); 190 191 static inline volatile u_int *pte_find(struct pmap *, vaddr_t); 192 static inline int pte_enter(struct pmap *, vaddr_t, u_int); 193 194 static inline int pmap_enter_pv(struct pmap *, vaddr_t, paddr_t, int); 195 static void pmap_remove_pv(struct pmap *, vaddr_t, paddr_t); 196 197 static inline void tlb_invalidate_entry(int); 198 199 static int ppc4xx_tlb_size_mask(size_t, int *, int *); 200 201 202 struct pv_entry * 203 pa_to_pv(paddr_t pa) 204 { 205 uvm_physseg_t bank; 206 psize_t pg; 207 208 bank = uvm_physseg_find(atop(pa), &pg); 209 if (bank == UVM_PHYSSEG_TYPE_INVALID) 210 return NULL; 211 return &uvm_physseg_get_pmseg(bank)->pvent[pg]; 212 } 213 214 static inline char * 215 pa_to_attr(paddr_t pa) 216 { 217 uvm_physseg_t bank; 218 psize_t pg; 219 220 bank = uvm_physseg_find(atop(pa), &pg); 221 if (bank == UVM_PHYSSEG_TYPE_INVALID) 222 return NULL; 223 return &uvm_physseg_get_pmseg(bank)->attrs[pg]; 224 } 225 226 /* 227 * Insert PTE into page table. 228 */ 229 static inline int 230 pte_enter(struct pmap *pm, vaddr_t va, u_int pte) 231 { 232 int seg = STIDX(va); 233 int ptn = PTIDX(va); 234 u_int oldpte; 235 236 if (!pm->pm_ptbl[seg]) { 237 /* Don't allocate a page to clear a non-existent mapping. */ 238 if (!pte) 239 return 1; 240 241 vaddr_t km = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 242 UVM_KMF_WIRED | UVM_KMF_ZERO | UVM_KMF_NOWAIT); 243 244 if (__predict_false(km == 0)) 245 return 0; 246 247 pm->pm_ptbl[seg] = (u_int *)km; 248 } 249 oldpte = pm->pm_ptbl[seg][ptn]; 250 pm->pm_ptbl[seg][ptn] = pte; 251 252 /* Flush entry. */ 253 ppc4xx_tlb_flush(va, pm->pm_ctx); 254 if (oldpte != pte) { 255 if (pte == 0) 256 pm->pm_stats.resident_count--; 257 else 258 pm->pm_stats.resident_count++; 259 } 260 return 1; 261 } 262 263 /* 264 * Get a pointer to a PTE in a page table. 265 */ 266 volatile u_int * 267 pte_find(struct pmap *pm, vaddr_t va) 268 { 269 int seg = STIDX(va); 270 int ptn = PTIDX(va); 271 272 if (pm->pm_ptbl[seg]) 273 return &pm->pm_ptbl[seg][ptn]; 274 275 return NULL; 276 } 277 278 /* 279 * This is called during initppc, before the system is really initialized. 280 */ 281 void 282 pmap_bootstrap(u_int kernelstart, u_int kernelend) 283 { 284 struct mem_region *mp, *mp1; 285 int cnt, i; 286 u_int s, e, sz; 287 288 tlbnext = tlb_nreserved; 289 290 /* 291 * Allocate the kernel page table at the end of 292 * kernel space so it's in the locked TTE. 293 */ 294 kernmap = (void *)kernelend; 295 296 /* 297 * Initialize kernel page table. 298 */ 299 for (i = 0; i < STSZ; i++) { 300 pmap_kernel()->pm_ptbl[i] = NULL; 301 } 302 ctxbusy[0] = ctxbusy[1] = pmap_kernel(); 303 304 /* 305 * Announce page-size to the VM-system 306 */ 307 uvmexp.pagesize = NBPG; 308 uvm_md_init(); 309 310 /* 311 * Get memory. 312 */ 313 mem_regions(&mem, &avail); 314 for (mp = mem; mp->size; mp++) { 315 physmem += btoc(mp->size); 316 printf("+%lx,",mp->size); 317 } 318 printf("\n"); 319 ppc4xx_tlb_init(); 320 /* 321 * Count the number of available entries. 322 */ 323 for (cnt = 0, mp = avail; mp->size; mp++) 324 cnt++; 325 326 /* 327 * Page align all regions. 328 * Non-page aligned memory isn't very interesting to us. 329 * Also, sort the entries for ascending addresses. 330 */ 331 kernelstart &= ~PGOFSET; 332 kernelend = (kernelend + PGOFSET) & ~PGOFSET; 333 for (mp = avail; mp->size; mp++) { 334 s = mp->start; 335 e = mp->start + mp->size; 336 printf("%08x-%08x -> ",s,e); 337 /* 338 * Check whether this region holds all of the kernel. 339 */ 340 if (s < kernelstart && e > kernelend) { 341 avail[cnt].start = kernelend; 342 avail[cnt++].size = e - kernelend; 343 e = kernelstart; 344 } 345 /* 346 * Look whether this regions starts within the kernel. 347 */ 348 if (s >= kernelstart && s < kernelend) { 349 if (e <= kernelend) 350 goto empty; 351 s = kernelend; 352 } 353 /* 354 * Now look whether this region ends within the kernel. 355 */ 356 if (e > kernelstart && e <= kernelend) { 357 if (s >= kernelstart) 358 goto empty; 359 e = kernelstart; 360 } 361 /* 362 * Now page align the start and size of the region. 363 */ 364 s = round_page(s); 365 e = trunc_page(e); 366 if (e < s) 367 e = s; 368 sz = e - s; 369 printf("%08x-%08x = %x\n",s,e,sz); 370 /* 371 * Check whether some memory is left here. 372 */ 373 if (sz == 0) { 374 empty: 375 memmove(mp, mp + 1, 376 (cnt - (mp - avail)) * sizeof *mp); 377 cnt--; 378 mp--; 379 continue; 380 } 381 /* 382 * Do an insertion sort. 383 */ 384 npgs += btoc(sz); 385 for (mp1 = avail; mp1 < mp; mp1++) 386 if (s < mp1->start) 387 break; 388 if (mp1 < mp) { 389 memmove(mp1 + 1, mp1, (char *)mp - (char *)mp1); 390 mp1->start = s; 391 mp1->size = sz; 392 } else { 393 mp->start = s; 394 mp->size = sz; 395 } 396 } 397 398 /* 399 * We cannot do pmap_steal_memory here, 400 * since we don't run with translation enabled yet. 401 */ 402 #ifndef MSGBUFADDR 403 /* 404 * allow for msgbuf 405 */ 406 sz = round_page(MSGBUFSIZE); 407 mp = NULL; 408 for (mp1 = avail; mp1->size; mp1++) 409 if (mp1->size >= sz) 410 mp = mp1; 411 if (mp == NULL) 412 panic("not enough memory?"); 413 414 npgs -= btoc(sz); 415 msgbuf_paddr = mp->start + mp->size - sz; 416 mp->size -= sz; 417 if (mp->size <= 0) 418 memmove(mp, mp + 1, (cnt - (mp - avail)) * sizeof *mp); 419 #endif 420 421 for (mp = avail; mp->size; mp++) 422 uvm_page_physload(atop(mp->start), atop(mp->start + mp->size), 423 atop(mp->start), atop(mp->start + mp->size), 424 VM_FREELIST_DEFAULT); 425 426 /* 427 * Initialize kernel pmap and hardware. 428 */ 429 /* Setup TLB pid allocator so it knows we alreadu using PID 1 */ 430 pmap_kernel()->pm_ctx = KERNEL_PID; 431 nextavail = avail->start; 432 433 pmap_bootstrap_done = 1; 434 } 435 436 /* 437 * Restrict given range to physical memory 438 * 439 * (Used by /dev/mem) 440 */ 441 void 442 pmap_real_memory(paddr_t *start, psize_t *size) 443 { 444 struct mem_region *mp; 445 446 for (mp = mem; mp->size; mp++) { 447 if (*start + *size > mp->start && 448 *start < mp->start + mp->size) { 449 if (*start < mp->start) { 450 *size -= mp->start - *start; 451 *start = mp->start; 452 } 453 if (*start + *size > mp->start + mp->size) 454 *size = mp->start + mp->size - *start; 455 return; 456 } 457 } 458 *size = 0; 459 } 460 461 /* 462 * Initialize anything else for pmap handling. 463 * Called during vm_init(). 464 */ 465 void 466 pmap_init(void) 467 { 468 struct pv_entry *pv; 469 vsize_t sz; 470 vaddr_t addr; 471 int i, s; 472 int bank; 473 char *attr; 474 475 sz = (vsize_t)((sizeof(struct pv_entry) + 1) * npgs); 476 sz = round_page(sz); 477 addr = uvm_km_alloc(kernel_map, sz, 0, UVM_KMF_WIRED | UVM_KMF_ZERO); 478 s = splvm(); 479 pv = pv_table = (struct pv_entry *)addr; 480 for (i = npgs; --i >= 0;) 481 pv++->pv_pm = NULL; 482 pmap_attrib = (char *)pv; 483 memset(pv, 0, npgs); 484 485 pv = pv_table; 486 attr = pmap_attrib; 487 for (bank = uvm_physseg_get_first(); 488 uvm_physseg_valid_p(bank); 489 bank = uvm_physseg_get_next(bank)) { 490 sz = uvm_physseg_get_end(bank) - uvm_physseg_get_start(bank); 491 uvm_physseg_get_pmseg(bank)->pvent = pv; 492 uvm_physseg_get_pmseg(bank)->attrs = attr; 493 pv += sz; 494 attr += sz; 495 } 496 497 pmap_initialized = 1; 498 splx(s); 499 500 /* Setup a pool for additional pvlist structures */ 501 pool_init(&pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pv_entry", NULL, 502 IPL_VM); 503 } 504 505 /* 506 * How much virtual space is available to the kernel? 507 */ 508 void 509 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 510 { 511 512 #if 0 513 /* 514 * Reserve one segment for kernel virtual memory 515 */ 516 *start = (vaddr_t)(KERNEL_SR << ADDR_SR_SHFT); 517 *end = *start + SEGMENT_LENGTH; 518 #else 519 *start = (vaddr_t) VM_MIN_KERNEL_ADDRESS; 520 *end = (vaddr_t) VM_MAX_KERNEL_ADDRESS; 521 #endif 522 } 523 524 #ifdef PMAP_GROWKERNEL 525 /* 526 * Preallocate kernel page tables to a specified VA. 527 * This simply loops through the first TTE for each 528 * page table from the beginning of the kernel pmap, 529 * reads the entry, and if the result is 530 * zero (either invalid entry or no page table) it stores 531 * a zero there, populating page tables in the process. 532 * This is not the most efficient technique but i don't 533 * expect it to be called that often. 534 */ 535 extern struct vm_page *vm_page_alloc1(void); 536 extern void vm_page_free1(struct vm_page *); 537 538 vaddr_t kbreak = VM_MIN_KERNEL_ADDRESS; 539 540 vaddr_t 541 pmap_growkernel(vaddr_t maxkvaddr) 542 { 543 int s; 544 int seg; 545 paddr_t pg; 546 struct pmap *pm = pmap_kernel(); 547 548 s = splvm(); 549 550 /* Align with the start of a page table */ 551 for (kbreak &= ~(PTMAP-1); kbreak < maxkvaddr; 552 kbreak += PTMAP) { 553 seg = STIDX(kbreak); 554 555 if (pte_find(pm, kbreak)) 556 continue; 557 558 if (uvm.page_init_done) { 559 pg = (paddr_t)VM_PAGE_TO_PHYS(vm_page_alloc1()); 560 } else { 561 if (!uvm_page_physget(&pg)) 562 panic("pmap_growkernel: no memory"); 563 } 564 if (!pg) 565 panic("pmap_growkernel: no pages"); 566 pmap_zero_page((paddr_t)pg); 567 568 /* XXX This is based on all phymem being addressable */ 569 pm->pm_ptbl[seg] = (u_int *)pg; 570 } 571 splx(s); 572 return kbreak; 573 } 574 575 /* 576 * vm_page_alloc1: 577 * 578 * Allocate and return a memory cell with no associated object. 579 */ 580 struct vm_page * 581 vm_page_alloc1(void) 582 { 583 struct vm_page *pg; 584 585 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE); 586 if (pg) { 587 pg->wire_count = 1; /* no mappings yet */ 588 pg->flags &= ~PG_BUSY; /* never busy */ 589 } 590 return pg; 591 } 592 593 /* 594 * vm_page_free1: 595 * 596 * Returns the given page to the free list, 597 * disassociating it with any VM object. 598 * 599 * Object and page must be locked prior to entry. 600 */ 601 void 602 vm_page_free1(struct vm_page *pg) 603 { 604 #ifdef DIAGNOSTIC 605 if (pg->flags != (PG_CLEAN|PG_FAKE)) { 606 printf("Freeing invalid page %p\n", pg); 607 printf("pa = %llx\n", (unsigned long long)VM_PAGE_TO_PHYS(pg)); 608 #ifdef DDB 609 Debugger(); 610 #endif 611 return; 612 } 613 #endif 614 pg->flags |= PG_BUSY; 615 pg->wire_count = 0; 616 uvm_pagefree(pg); 617 } 618 #endif 619 620 /* 621 * Create and return a physical map. 622 */ 623 struct pmap * 624 pmap_create(void) 625 { 626 struct pmap *pm; 627 628 pm = kmem_alloc(sizeof(*pm), KM_SLEEP); 629 memset(pm, 0, sizeof *pm); 630 pm->pm_refs = 1; 631 return pm; 632 } 633 634 /* 635 * Add a reference to the given pmap. 636 */ 637 void 638 pmap_reference(struct pmap *pm) 639 { 640 641 pm->pm_refs++; 642 } 643 644 /* 645 * Retire the given pmap from service. 646 * Should only be called if the map contains no valid mappings. 647 */ 648 void 649 pmap_destroy(struct pmap *pm) 650 { 651 int i; 652 653 if (--pm->pm_refs > 0) { 654 return; 655 } 656 KASSERT(pm->pm_stats.resident_count == 0); 657 KASSERT(pm->pm_stats.wired_count == 0); 658 for (i = 0; i < STSZ; i++) 659 if (pm->pm_ptbl[i]) { 660 uvm_km_free(kernel_map, (vaddr_t)pm->pm_ptbl[i], 661 PAGE_SIZE, UVM_KMF_WIRED); 662 pm->pm_ptbl[i] = NULL; 663 } 664 if (pm->pm_ctx) 665 ctx_free(pm); 666 kmem_free(pm, sizeof(*pm)); 667 } 668 669 /* 670 * Copy the range specified by src_addr/len 671 * from the source map to the range dst_addr/len 672 * in the destination map. 673 * 674 * This routine is only advisory and need not do anything. 675 */ 676 void 677 pmap_copy(struct pmap *dst_pmap, struct pmap *src_pmap, vaddr_t dst_addr, 678 vsize_t len, vaddr_t src_addr) 679 { 680 } 681 682 /* 683 * Require that all active physical maps contain no 684 * incorrect entries NOW. 685 */ 686 void 687 pmap_update(struct pmap *pmap) 688 { 689 } 690 691 /* 692 * Fill the given physical page with zeroes. 693 */ 694 void 695 pmap_zero_page(paddr_t pa) 696 { 697 698 #ifdef PPC_4XX_NOCACHE 699 memset((void *)pa, 0, PAGE_SIZE); 700 #else 701 int i; 702 703 for (i = PAGE_SIZE/CACHELINESIZE; i > 0; i--) { 704 __asm volatile ("dcbz 0,%0" :: "r"(pa)); 705 pa += CACHELINESIZE; 706 } 707 #endif 708 } 709 710 /* 711 * Copy the given physical source page to its destination. 712 */ 713 void 714 pmap_copy_page(paddr_t src, paddr_t dst) 715 { 716 717 memcpy((void *)dst, (void *)src, PAGE_SIZE); 718 dcache_wbinv_page(dst); 719 } 720 721 /* 722 * This returns != 0 on success. 723 */ 724 static inline int 725 pmap_enter_pv(struct pmap *pm, vaddr_t va, paddr_t pa, int flags) 726 { 727 struct pv_entry *pv, *npv = NULL; 728 int s; 729 730 if (!pmap_initialized) 731 return 0; 732 733 s = splvm(); 734 pv = pa_to_pv(pa); 735 if (!pv->pv_pm) { 736 /* 737 * No entries yet, use header as the first entry. 738 */ 739 pv->pv_va = va; 740 pv->pv_pm = pm; 741 pv->pv_next = NULL; 742 } else { 743 /* 744 * There is at least one other VA mapping this page. 745 * Place this entry after the header. 746 */ 747 npv = pool_get(&pv_pool, PR_NOWAIT); 748 if (npv == NULL) { 749 if ((flags & PMAP_CANFAIL) == 0) 750 panic("pmap_enter_pv: failed"); 751 splx(s); 752 return 0; 753 } 754 npv->pv_va = va; 755 npv->pv_pm = pm; 756 npv->pv_next = pv->pv_next; 757 pv->pv_next = npv; 758 pv = npv; 759 } 760 if (flags & PMAP_WIRED) { 761 PV_WIRE(pv); 762 pm->pm_stats.wired_count++; 763 } 764 splx(s); 765 return 1; 766 } 767 768 static void 769 pmap_remove_pv(struct pmap *pm, vaddr_t va, paddr_t pa) 770 { 771 struct pv_entry *pv, *npv; 772 773 /* 774 * Remove from the PV table. 775 */ 776 pv = pa_to_pv(pa); 777 if (!pv) 778 return; 779 780 /* 781 * If it is the first entry on the list, it is actually 782 * in the header and we must copy the following entry up 783 * to the header. Otherwise we must search the list for 784 * the entry. In either case we free the now unused entry. 785 */ 786 if (pm == pv->pv_pm && PV_CMPVA(va, pv)) { 787 if (PV_ISWIRED(pv)) { 788 pm->pm_stats.wired_count--; 789 } 790 if ((npv = pv->pv_next)) { 791 *pv = *npv; 792 pool_put(&pv_pool, npv); 793 } else 794 pv->pv_pm = NULL; 795 } else { 796 for (; (npv = pv->pv_next) != NULL; pv = npv) 797 if (pm == npv->pv_pm && PV_CMPVA(va, npv)) 798 break; 799 if (npv) { 800 pv->pv_next = npv->pv_next; 801 if (PV_ISWIRED(npv)) { 802 pm->pm_stats.wired_count--; 803 } 804 pool_put(&pv_pool, npv); 805 } 806 } 807 } 808 809 /* 810 * Insert physical page at pa into the given pmap at virtual address va. 811 */ 812 int 813 pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 814 { 815 int s; 816 u_int tte; 817 bool managed; 818 819 /* 820 * Have to remove any existing mapping first. 821 */ 822 pmap_remove(pm, va, va + PAGE_SIZE); 823 824 if (flags & PMAP_WIRED) 825 flags |= prot; 826 827 managed = uvm_pageismanaged(pa); 828 829 /* 830 * Generate TTE. 831 */ 832 tte = TTE_PA(pa); 833 /* XXXX -- need to support multiple page sizes. */ 834 tte |= TTE_SZ_16K; 835 #ifdef DIAGNOSTIC 836 if ((flags & (PMAP_NOCACHE | PME_WRITETHROUG)) == 837 (PMAP_NOCACHE | PME_WRITETHROUG)) 838 panic("pmap_enter: uncached & writethrough"); 839 #endif 840 if (flags & PMAP_NOCACHE) 841 /* Must be I/O mapping */ 842 tte |= TTE_I | TTE_G; 843 #ifdef PPC_4XX_NOCACHE 844 tte |= TTE_I; 845 #else 846 else if (flags & PME_WRITETHROUG) 847 /* Uncached and writethrough are not compatible */ 848 tte |= TTE_W; 849 #endif 850 if (pm == pmap_kernel()) 851 tte |= TTE_ZONE(ZONE_PRIV); 852 else 853 tte |= TTE_ZONE(ZONE_USER); 854 855 if (flags & VM_PROT_WRITE) 856 tte |= TTE_WR; 857 858 if (flags & VM_PROT_EXECUTE) 859 tte |= TTE_EX; 860 861 /* 862 * Now record mapping for later back-translation. 863 */ 864 if (pmap_initialized && managed) { 865 char *attr; 866 867 if (!pmap_enter_pv(pm, va, pa, flags)) { 868 /* Could not enter pv on a managed page */ 869 return ENOMEM; 870 } 871 872 /* Now set attributes. */ 873 attr = pa_to_attr(pa); 874 #ifdef DIAGNOSTIC 875 if (!attr) 876 panic("managed but no attr"); 877 #endif 878 if (flags & VM_PROT_ALL) 879 *attr |= PMAP_ATTR_REF; 880 if (flags & VM_PROT_WRITE) 881 *attr |= PMAP_ATTR_CHG; 882 } 883 884 s = splvm(); 885 886 /* Insert page into page table. */ 887 if (__predict_false(!pte_enter(pm, va, tte))) { 888 if (__predict_false((flags & PMAP_CANFAIL) == 0)) 889 panic("%s: pte_enter", __func__); 890 splx(s); 891 return ENOMEM; 892 } 893 894 /* If this is a real fault, enter it in the tlb */ 895 if (tte && ((flags & PMAP_WIRED) == 0)) { 896 int s2 = splhigh(); 897 ppc4xx_tlb_enter(pm->pm_ctx, va, tte); 898 splx(s2); 899 } 900 splx(s); 901 902 /* Flush the real memory from the instruction cache. */ 903 if ((prot & VM_PROT_EXECUTE) && (tte & TTE_I) == 0) 904 __syncicache((void *)pa, PAGE_SIZE); 905 906 return 0; 907 } 908 909 void 910 pmap_unwire(struct pmap *pm, vaddr_t va) 911 { 912 struct pv_entry *pv; 913 paddr_t pa; 914 int s; 915 916 if (!pmap_extract(pm, va, &pa)) { 917 return; 918 } 919 920 pv = pa_to_pv(pa); 921 if (!pv) 922 return; 923 924 s = splvm(); 925 while (pv != NULL) { 926 if (pm == pv->pv_pm && PV_CMPVA(va, pv)) { 927 if (PV_ISWIRED(pv)) { 928 PV_UNWIRE(pv); 929 pm->pm_stats.wired_count--; 930 } 931 break; 932 } 933 pv = pv->pv_next; 934 } 935 splx(s); 936 } 937 938 void 939 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 940 { 941 int s; 942 u_int tte; 943 struct pmap *pm = pmap_kernel(); 944 945 /* 946 * Generate TTE. 947 * 948 * XXXX 949 * 950 * Since the kernel does not handle execution privileges properly, 951 * we will handle read and execute permissions together. 952 */ 953 tte = 0; 954 if (prot & VM_PROT_ALL) { 955 956 tte = TTE_PA(pa) | TTE_EX | TTE_ZONE(ZONE_PRIV); 957 /* XXXX -- need to support multiple page sizes. */ 958 tte |= TTE_SZ_16K; 959 #ifdef DIAGNOSTIC 960 if ((flags & (PMAP_NOCACHE | PME_WRITETHROUG)) == 961 (PMAP_NOCACHE | PME_WRITETHROUG)) 962 panic("pmap_kenter_pa: uncached & writethrough"); 963 #endif 964 if (flags & PMAP_NOCACHE) 965 /* Must be I/O mapping */ 966 tte |= TTE_I | TTE_G; 967 #ifdef PPC_4XX_NOCACHE 968 tte |= TTE_I; 969 #else 970 else if (prot & PME_WRITETHROUG) 971 /* Uncached and writethrough are not compatible */ 972 tte |= TTE_W; 973 #endif 974 if (prot & VM_PROT_WRITE) 975 tte |= TTE_WR; 976 } 977 978 s = splvm(); 979 980 /* Insert page into page table. */ 981 if (__predict_false(!pte_enter(pm, va, tte))) 982 panic("%s: pte_enter", __func__); 983 splx(s); 984 } 985 986 void 987 pmap_kremove(vaddr_t va, vsize_t len) 988 { 989 990 while (len > 0) { 991 (void)pte_enter(pmap_kernel(), va, 0); /* never fail */ 992 va += PAGE_SIZE; 993 len -= PAGE_SIZE; 994 } 995 } 996 997 /* 998 * Remove the given range of mapping entries. 999 */ 1000 void 1001 pmap_remove(struct pmap *pm, vaddr_t va, vaddr_t endva) 1002 { 1003 int s; 1004 paddr_t pa; 1005 volatile u_int *ptp; 1006 1007 s = splvm(); 1008 while (va < endva) { 1009 1010 if ((ptp = pte_find(pm, va)) && (pa = *ptp)) { 1011 pa = TTE_PA(pa); 1012 pmap_remove_pv(pm, va, pa); 1013 *ptp = 0; 1014 ppc4xx_tlb_flush(va, pm->pm_ctx); 1015 pm->pm_stats.resident_count--; 1016 } 1017 va += PAGE_SIZE; 1018 } 1019 1020 splx(s); 1021 } 1022 1023 /* 1024 * Get the physical page address for the given pmap/virtual address. 1025 */ 1026 bool 1027 pmap_extract(struct pmap *pm, vaddr_t va, paddr_t *pap) 1028 { 1029 int seg = STIDX(va); 1030 int ptn = PTIDX(va); 1031 u_int pa = 0; 1032 int s; 1033 1034 s = splvm(); 1035 if (pm->pm_ptbl[seg] && (pa = pm->pm_ptbl[seg][ptn]) && pap) { 1036 *pap = TTE_PA(pa) | (va & PGOFSET); 1037 } 1038 splx(s); 1039 return pa != 0; 1040 } 1041 1042 /* 1043 * Lower the protection on the specified range of this pmap. 1044 * 1045 * There are only two cases: either the protection is going to 0, 1046 * or it is going to read-only. 1047 */ 1048 void 1049 pmap_protect(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1050 { 1051 volatile u_int *ptp; 1052 int s, bic; 1053 1054 if ((prot & VM_PROT_READ) == 0) { 1055 pmap_remove(pm, sva, eva); 1056 return; 1057 } 1058 bic = 0; 1059 if ((prot & VM_PROT_WRITE) == 0) { 1060 bic |= TTE_WR; 1061 } 1062 if ((prot & VM_PROT_EXECUTE) == 0) { 1063 bic |= TTE_EX; 1064 } 1065 if (bic == 0) { 1066 return; 1067 } 1068 s = splvm(); 1069 while (sva < eva) { 1070 if ((ptp = pte_find(pm, sva)) != NULL) { 1071 *ptp &= ~bic; 1072 ppc4xx_tlb_flush(sva, pm->pm_ctx); 1073 } 1074 sva += PAGE_SIZE; 1075 } 1076 splx(s); 1077 } 1078 1079 bool 1080 pmap_check_attr(struct vm_page *pg, u_int mask, int clear) 1081 { 1082 paddr_t pa; 1083 char *attr; 1084 int s, rv; 1085 1086 /* 1087 * First modify bits in cache. 1088 */ 1089 pa = VM_PAGE_TO_PHYS(pg); 1090 attr = pa_to_attr(pa); 1091 if (attr == NULL) 1092 return false; 1093 1094 s = splvm(); 1095 rv = ((*attr & mask) != 0); 1096 if (clear) { 1097 *attr &= ~mask; 1098 pmap_page_protect(pg, mask == PMAP_ATTR_CHG ? VM_PROT_READ : 0); 1099 } 1100 splx(s); 1101 return rv; 1102 } 1103 1104 1105 /* 1106 * Lower the protection on the specified physical page. 1107 * 1108 * There are only two cases: either the protection is going to 0, 1109 * or it is going to read-only. 1110 */ 1111 void 1112 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 1113 { 1114 paddr_t pa = VM_PAGE_TO_PHYS(pg); 1115 vaddr_t va; 1116 struct pv_entry *pvh, *pv, *npv; 1117 struct pmap *pm; 1118 1119 pvh = pa_to_pv(pa); 1120 if (pvh == NULL) 1121 return; 1122 1123 /* Handle extra pvs which may be deleted in the operation */ 1124 for (pv = pvh->pv_next; pv; pv = npv) { 1125 npv = pv->pv_next; 1126 1127 pm = pv->pv_pm; 1128 va = PV_VA(pv); 1129 pmap_protect(pm, va, va + PAGE_SIZE, prot); 1130 } 1131 /* Now check the head pv */ 1132 if (pvh->pv_pm) { 1133 pv = pvh; 1134 pm = pv->pv_pm; 1135 va = PV_VA(pv); 1136 pmap_protect(pm, va, va + PAGE_SIZE, prot); 1137 } 1138 } 1139 1140 /* 1141 * Activate the address space for the specified process. If the process 1142 * is the current process, load the new MMU context. 1143 */ 1144 void 1145 pmap_activate(struct lwp *l) 1146 { 1147 #if 0 1148 struct pcb *pcb = lwp_getpcb(l); 1149 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 1150 1151 /* 1152 * XXX Normally performed in cpu_lwp_fork(). 1153 */ 1154 printf("pmap_activate(%p), pmap=%p\n",l,pmap); 1155 pcb->pcb_pm = pmap; 1156 #endif 1157 } 1158 1159 /* 1160 * Deactivate the specified process's address space. 1161 */ 1162 void 1163 pmap_deactivate(struct lwp *l) 1164 { 1165 } 1166 1167 /* 1168 * Synchronize caches corresponding to [addr, addr+len) in p. 1169 */ 1170 void 1171 pmap_procwr(struct proc *p, vaddr_t va, size_t len) 1172 { 1173 struct pmap *pm = p->p_vmspace->vm_map.pmap; 1174 1175 if (__predict_true(p == curproc)) { 1176 int msr, ctx, opid; 1177 1178 /* 1179 * Take it easy! TLB miss handler takes care of us. 1180 */ 1181 1182 /* 1183 * Need to turn off IMMU and switch to user context. 1184 * (icbi uses DMMU). 1185 */ 1186 1187 if (!(ctx = pm->pm_ctx)) { 1188 /* No context -- assign it one */ 1189 ctx_alloc(pm); 1190 ctx = pm->pm_ctx; 1191 } 1192 1193 __asm volatile( 1194 "mfmsr %0;" 1195 "li %1,0x20;" /* Turn off IMMU */ 1196 "andc %1,%0,%1;" 1197 "ori %1,%1,0x10;" /* Turn on DMMU for sure */ 1198 "mtmsr %1;" 1199 "isync;" 1200 "mfpid %1;" 1201 "mtpid %2;" 1202 "isync;" 1203 "1:" 1204 "dcbst 0,%3;" 1205 "icbi 0,%3;" 1206 "add %3,%3,%5;" 1207 "sub. %4,%4,%5;" 1208 "bge 1b;" 1209 "sync;" 1210 "mtpid %1;" 1211 "mtmsr %0;" 1212 "isync;" 1213 : "=&r" (msr), "=&r" (opid) 1214 : "r" (ctx), "r" (va), "r" (len), "r" (CACHELINESIZE)); 1215 } else { 1216 paddr_t pa; 1217 vaddr_t tva, eva; 1218 int tlen; 1219 1220 /* 1221 * For p != curproc, we cannot rely upon TLB miss handler in 1222 * user context. Therefore, extract pa and operate againt it. 1223 * 1224 * Note that va below VM_MIN_KERNEL_ADDRESS is reserved for 1225 * direct mapping. 1226 */ 1227 1228 for (tva = va; len > 0; tva = eva, len -= tlen) { 1229 eva = uimin(tva + len, trunc_page(tva + PAGE_SIZE)); 1230 tlen = eva - tva; 1231 if (!pmap_extract(pm, tva, &pa)) { 1232 /* XXX should be already unmapped */ 1233 continue; 1234 } 1235 __syncicache((void *)pa, tlen); 1236 } 1237 } 1238 } 1239 1240 static inline void 1241 tlb_invalidate_entry(int i) 1242 { 1243 #ifdef PMAP_TLBDEBUG 1244 /* 1245 * Clear only TLBHI[V] bit so that we can track invalidated entry. 1246 */ 1247 register_t msr, pid, hi; 1248 1249 KASSERT(mfspr(SPR_PID) == KERNEL_PID); 1250 1251 __asm volatile( 1252 "mfmsr %0;" 1253 "li %1,0;" 1254 "mtmsr %1;" 1255 "mfpid %1;" 1256 "tlbre %2,%3,0;" 1257 "andc %2,%2,%4;" 1258 "tlbwe %2,%3,0;" 1259 "mtpid %1;" 1260 "mtmsr %0;" 1261 "isync;" 1262 : "=&r" (msr), "=&r" (pid), "=&r" (hi) 1263 : "r" (i), "r" (TLB_VALID)); 1264 #else 1265 /* 1266 * Just clear entire TLBHI register. 1267 */ 1268 __asm volatile( 1269 "tlbwe %0,%1,0;" 1270 "isync;" 1271 : : "r" (0), "r" (i)); 1272 #endif 1273 1274 tlb_info[i].ti_ctx = 0; 1275 tlb_info[i].ti_flags = 0; 1276 } 1277 1278 /* This has to be done in real mode !!! */ 1279 void 1280 ppc4xx_tlb_flush(vaddr_t va, int pid) 1281 { 1282 u_long i, found; 1283 u_long msr; 1284 1285 /* If there's no context then it can't be mapped. */ 1286 if (!pid) 1287 return; 1288 1289 __asm volatile( 1290 "mfpid %1;" /* Save PID */ 1291 "mfmsr %2;" /* Save MSR */ 1292 "li %0,0;" /* Now clear MSR */ 1293 "mtmsr %0;" 1294 "isync;" 1295 "mtpid %4;" /* Set PID */ 1296 "isync;" 1297 "tlbsx. %0,0,%3;" /* Search TLB */ 1298 "isync;" 1299 "mtpid %1;" /* Restore PID */ 1300 "mtmsr %2;" /* Restore MSR */ 1301 "isync;" 1302 "li %1,1;" 1303 "beq 1f;" 1304 "li %1,0;" 1305 "1:" 1306 : "=&r" (i), "=&r" (found), "=&r" (msr) 1307 : "r" (va), "r" (pid)); 1308 if (found && !TLB_LOCKED(i)) { 1309 /* Now flush translation */ 1310 tlb_invalidate_entry(i); 1311 tlbnext = i; 1312 /* Successful flushes */ 1313 tlbflush_ev.ev_count++; 1314 } 1315 } 1316 1317 void 1318 ppc4xx_tlb_flush_all(void) 1319 { 1320 u_long i; 1321 1322 for (i = 0; i < NTLB; i++) 1323 if (!TLB_LOCKED(i)) 1324 tlb_invalidate_entry(i); 1325 1326 __asm volatile("isync"); 1327 } 1328 1329 /* Find a TLB entry to evict. */ 1330 static int 1331 ppc4xx_tlb_find_victim(void) 1332 { 1333 int flags; 1334 1335 for (;;) { 1336 if (++tlbnext >= NTLB) 1337 tlbnext = tlb_nreserved; 1338 flags = tlb_info[tlbnext].ti_flags; 1339 if (!(flags & TLBF_USED) || 1340 (flags & (TLBF_LOCKED | TLBF_REF)) == 0) { 1341 u_long va, stack = (u_long)&va; 1342 1343 if (!((tlb_info[tlbnext].ti_va ^ stack) & (~PGOFSET)) && 1344 (tlb_info[tlbnext].ti_ctx == KERNEL_PID) && 1345 (flags & TLBF_USED)) { 1346 /* Kernel stack page */ 1347 flags |= TLBF_REF; 1348 tlb_info[tlbnext].ti_flags = flags; 1349 } else { 1350 /* Found it! */ 1351 return tlbnext; 1352 } 1353 } else { 1354 tlb_info[tlbnext].ti_flags = (flags & ~TLBF_REF); 1355 } 1356 } 1357 } 1358 1359 void 1360 ppc4xx_tlb_enter(int ctx, vaddr_t va, u_int pte) 1361 { 1362 u_long th, tl, idx; 1363 int msr, pid; 1364 paddr_t pa; 1365 int sz; 1366 1367 tlbenter_ev.ev_count++; 1368 1369 sz = (pte & TTE_SZ_MASK) >> TTE_SZ_SHIFT; 1370 pa = (pte & TTE_RPN_MASK(sz)); 1371 th = (va & TLB_EPN_MASK) | (sz << TLB_SIZE_SHFT) | TLB_VALID; 1372 tl = (pte & ~TLB_RPN_MASK) | pa; 1373 tl |= ppc4xx_tlbflags(va, pa); 1374 1375 idx = ppc4xx_tlb_find_victim(); 1376 1377 #ifdef DIAGNOSTIC 1378 if ((idx < tlb_nreserved) || (idx >= NTLB) || (idx & 63) == 0) { 1379 panic("ppc4xx_tlb_enter: replacing entry %ld", idx); 1380 } 1381 #endif 1382 1383 tlb_info[idx].ti_va = (va & TLB_EPN_MASK); 1384 tlb_info[idx].ti_ctx = ctx; 1385 tlb_info[idx].ti_flags = TLBF_USED | TLBF_REF; 1386 1387 __asm volatile( 1388 "mfmsr %0;" /* Save MSR */ 1389 "li %1,0;" 1390 "mtmsr %1;" /* Clear MSR */ 1391 "isync;" 1392 "tlbwe %1,%3,0;" /* Invalidate old entry. */ 1393 "mfpid %1;" /* Save old PID */ 1394 "mtpid %2;" /* Load translation ctx */ 1395 "isync;" 1396 "tlbwe %4,%3,1; tlbwe %5,%3,0;" /* Set TLB */ 1397 "isync;" 1398 "mtpid %1; mtmsr %0;" /* Restore PID and MSR */ 1399 "isync;" 1400 : "=&r" (msr), "=&r" (pid) 1401 : "r" (ctx), "r" (idx), "r" (tl), "r" (th)); 1402 } 1403 1404 void 1405 ppc4xx_tlb_init(void) 1406 { 1407 int i; 1408 1409 /* Mark reserved TLB entries */ 1410 for (i = 0; i < tlb_nreserved; i++) { 1411 tlb_info[i].ti_flags = TLBF_LOCKED | TLBF_USED; 1412 tlb_info[i].ti_ctx = KERNEL_PID; 1413 } 1414 1415 /* Setup security zones */ 1416 /* Z0 - accessible by kernel only if TLB entry permissions allow 1417 * Z1,Z2 - access is controlled by TLB entry permissions 1418 * Z3 - full access regardless of TLB entry permissions 1419 */ 1420 1421 __asm volatile( 1422 "mtspr %0,%1;" 1423 "isync;" 1424 :: "K"(SPR_ZPR), "r" (0x1b000000)); 1425 } 1426 1427 /* 1428 * ppc4xx_tlb_size_mask: 1429 * 1430 * Roundup size to supported page size, return TLBHI mask and real size. 1431 */ 1432 static int 1433 ppc4xx_tlb_size_mask(size_t size, int *mask, int *rsiz) 1434 { 1435 int i; 1436 1437 for (i = 0; i < __arraycount(tlbsize); i++) 1438 if (size <= tlbsize[i]) { 1439 *mask = (i << TLB_SIZE_SHFT); 1440 *rsiz = tlbsize[i]; 1441 return 0; 1442 } 1443 return EINVAL; 1444 } 1445 1446 /* 1447 * ppc4xx_tlb_mapiodev: 1448 * 1449 * Lookup virtual address of mapping previously entered via 1450 * ppc4xx_tlb_reserve. Search TLB directly so that we don't 1451 * need to waste extra storage for reserved mappings. Note 1452 * that reading TLBHI also sets PID, but all reserved mappings 1453 * use KERNEL_PID, so the side effect is nil. 1454 */ 1455 void * 1456 ppc4xx_tlb_mapiodev(paddr_t base, psize_t len) 1457 { 1458 paddr_t pa; 1459 vaddr_t va; 1460 u_int lo, hi, sz; 1461 int i; 1462 1463 /* tlb_nreserved is only allowed to grow, so this is safe. */ 1464 for (i = 0; i < tlb_nreserved; i++) { 1465 __asm volatile ( 1466 " tlbre %0,%2,1 \n" /* TLBLO */ 1467 " tlbre %1,%2,0 \n" /* TLBHI */ 1468 : "=&r" (lo), "=&r" (hi) 1469 : "r" (i)); 1470 1471 KASSERT(hi & TLB_VALID); 1472 KASSERT(mfspr(SPR_PID) == KERNEL_PID); 1473 1474 pa = (lo & TLB_RPN_MASK); 1475 if (base < pa) 1476 continue; 1477 1478 sz = tlbsize[(hi & TLB_SIZE_MASK) >> TLB_SIZE_SHFT]; 1479 if ((base + len) > (pa + sz)) 1480 continue; 1481 1482 va = (hi & TLB_EPN_MASK) + (base & (sz - 1)); /* sz = 2^n */ 1483 return (void *)va; 1484 } 1485 1486 return NULL; 1487 } 1488 1489 /* 1490 * ppc4xx_tlb_reserve: 1491 * 1492 * Map physical range to kernel virtual chunk via reserved TLB entry. 1493 */ 1494 void 1495 ppc4xx_tlb_reserve(paddr_t pa, vaddr_t va, size_t size, int flags) 1496 { 1497 u_int lo, hi; 1498 int szmask, rsize; 1499 1500 /* Called before pmap_bootstrap(), va outside kernel space. */ 1501 KASSERT(va < VM_MIN_KERNEL_ADDRESS || va >= VM_MAX_KERNEL_ADDRESS); 1502 KASSERT(! pmap_bootstrap_done); 1503 KASSERT(tlb_nreserved < NTLB); 1504 1505 /* Resolve size. */ 1506 if (ppc4xx_tlb_size_mask(size, &szmask, &rsize) != 0) 1507 panic("ppc4xx_tlb_reserve: entry %d, %zuB too large", 1508 size, tlb_nreserved); 1509 1510 /* Real size will be power of two >= 1024, so this is OK. */ 1511 pa &= ~(rsize - 1); /* RPN */ 1512 va &= ~(rsize - 1); /* EPN */ 1513 1514 lo = pa | TLB_WR | flags; 1515 hi = va | TLB_VALID | szmask; 1516 1517 #ifdef PPC_4XX_NOCACHE 1518 lo |= TLB_I; 1519 #endif 1520 1521 __asm volatile( 1522 " tlbwe %1,%0,1 \n" /* write TLBLO */ 1523 " tlbwe %2,%0,0 \n" /* write TLBHI */ 1524 " isync \n" 1525 : : "r" (tlb_nreserved), "r" (lo), "r" (hi)); 1526 1527 tlb_nreserved++; 1528 } 1529 1530 /* 1531 * We should pass the ctx in from trap code. 1532 */ 1533 int 1534 pmap_tlbmiss(vaddr_t va, int ctx) 1535 { 1536 volatile u_int *pte; 1537 u_long tte; 1538 1539 tlbmiss_ev.ev_count++; 1540 1541 /* 1542 * We will reserve 0 upto VM_MIN_KERNEL_ADDRESS for va == pa mappings. 1543 * Physical RAM is expected to live in this range, care must be taken 1544 * to not clobber 0 upto ${physmem} with device mappings in machdep 1545 * code. 1546 */ 1547 if (ctx != KERNEL_PID || 1548 (va >= VM_MIN_KERNEL_ADDRESS && va < VM_MAX_KERNEL_ADDRESS)) { 1549 pte = pte_find((struct pmap *)__UNVOLATILE(ctxbusy[ctx]), va); 1550 if (pte == NULL) { 1551 /* Map unmanaged addresses directly for kernel access */ 1552 return 1; 1553 } 1554 tte = *pte; 1555 if (tte == 0) { 1556 return 1; 1557 } 1558 } else { 1559 /* Create a 16MB writable mapping. */ 1560 #ifdef PPC_4XX_NOCACHE 1561 tte = TTE_PA(va) | TTE_ZONE(ZONE_PRIV) | TTE_SZ_16M | TTE_I |TTE_WR; 1562 #else 1563 tte = TTE_PA(va) | TTE_ZONE(ZONE_PRIV) | TTE_SZ_16M | TTE_WR; 1564 #endif 1565 } 1566 ppc4xx_tlb_enter(ctx, va, tte); 1567 1568 return 0; 1569 } 1570 1571 /* 1572 * Flush all the entries matching a context from the TLB. 1573 */ 1574 static int 1575 ctx_flush(int cnum) 1576 { 1577 int i; 1578 1579 /* We gotta steal this context */ 1580 for (i = tlb_nreserved; i < NTLB; i++) { 1581 if (tlb_info[i].ti_ctx == cnum) { 1582 /* Can't steal ctx if it has a locked entry. */ 1583 if (TLB_LOCKED(i)) { 1584 #ifdef DIAGNOSTIC 1585 printf("ctx_flush: can't invalidate " 1586 "locked mapping %d " 1587 "for context %d\n", i, cnum); 1588 #ifdef DDB 1589 Debugger(); 1590 #endif 1591 #endif 1592 return 1; 1593 } 1594 #ifdef DIAGNOSTIC 1595 if (i < tlb_nreserved) 1596 panic("TLB entry %d not locked", i); 1597 #endif 1598 /* 1599 * Invalidate particular TLB entry regardless of 1600 * locked status 1601 */ 1602 tlb_invalidate_entry(i); 1603 } 1604 } 1605 return 0; 1606 } 1607 1608 /* 1609 * Allocate a context. If necessary, steal one from someone else. 1610 * 1611 * The new context is flushed from the TLB before returning. 1612 */ 1613 int 1614 ctx_alloc(struct pmap *pm) 1615 { 1616 int s, cnum; 1617 static int next = MINCTX; 1618 1619 if (pm == pmap_kernel()) { 1620 #ifdef DIAGNOSTIC 1621 printf("ctx_alloc: kernel pmap!\n"); 1622 #endif 1623 return 0; 1624 } 1625 s = splvm(); 1626 1627 /* Find a likely context. */ 1628 cnum = next; 1629 do { 1630 if ((++cnum) >= NUMCTX) 1631 cnum = MINCTX; 1632 } while (ctxbusy[cnum] != NULL && cnum != next); 1633 1634 /* Now clean it out */ 1635 oops: 1636 if (cnum < MINCTX) 1637 cnum = MINCTX; /* Never steal ctx 0 or 1 */ 1638 if (ctx_flush(cnum)) { 1639 /* oops -- something's wired. */ 1640 if ((++cnum) >= NUMCTX) 1641 cnum = MINCTX; 1642 goto oops; 1643 } 1644 1645 if (ctxbusy[cnum]) { 1646 #ifdef DEBUG 1647 /* We should identify this pmap and clear it */ 1648 printf("Warning: stealing context %d\n", cnum); 1649 #endif 1650 ctxbusy[cnum]->pm_ctx = 0; 1651 } 1652 ctxbusy[cnum] = pm; 1653 next = cnum; 1654 splx(s); 1655 pm->pm_ctx = cnum; 1656 1657 return cnum; 1658 } 1659 1660 /* 1661 * Give away a context. 1662 */ 1663 void 1664 ctx_free(struct pmap *pm) 1665 { 1666 int oldctx; 1667 1668 oldctx = pm->pm_ctx; 1669 1670 if (oldctx == 0) 1671 panic("ctx_free: freeing kernel context"); 1672 #ifdef DIAGNOSTIC 1673 if (ctxbusy[oldctx] == 0) 1674 printf("ctx_free: freeing free context %d\n", oldctx); 1675 if (ctxbusy[oldctx] != pm) { 1676 printf("ctx_free: freeing someone esle's context\n " 1677 "ctxbusy[%d] = %p, pm->pm_ctx = %p\n", 1678 oldctx, (void *)(u_long)ctxbusy[oldctx], pm); 1679 #ifdef DDB 1680 Debugger(); 1681 #endif 1682 } 1683 #endif 1684 /* We should verify it has not been stolen and reallocated... */ 1685 ctxbusy[oldctx] = NULL; 1686 ctx_flush(oldctx); 1687 } 1688 1689 1690 #ifdef DEBUG 1691 /* 1692 * Test ref/modify handling. 1693 */ 1694 void pmap_testout(void); 1695 void 1696 pmap_testout(void) 1697 { 1698 vaddr_t va; 1699 volatile int *loc; 1700 int val = 0; 1701 paddr_t pa; 1702 struct vm_page *pg; 1703 int ref, mod; 1704 1705 /* Allocate a page */ 1706 va = (vaddr_t)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 1707 UVM_KMF_WIRED | UVM_KMF_ZERO); 1708 loc = (int*)va; 1709 1710 pmap_extract(pmap_kernel(), va, &pa); 1711 pg = PHYS_TO_VM_PAGE(pa); 1712 pmap_unwire(pmap_kernel(), va); 1713 1714 pmap_kremove(va, PAGE_SIZE); 1715 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1716 pmap_update(pmap_kernel()); 1717 1718 /* Now clear reference and modify */ 1719 ref = pmap_clear_reference(pg); 1720 mod = pmap_clear_modify(pg); 1721 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1722 (void *)(u_long)va, (long)pa, 1723 ref, mod); 1724 1725 /* Check it's properly cleared */ 1726 ref = pmap_is_referenced(pg); 1727 mod = pmap_is_modified(pg); 1728 printf("Checking cleared page: ref %d, mod %d\n", 1729 ref, mod); 1730 1731 /* Reference page */ 1732 val = *loc; 1733 1734 ref = pmap_is_referenced(pg); 1735 mod = pmap_is_modified(pg); 1736 printf("Referenced page: ref %d, mod %d val %x\n", 1737 ref, mod, val); 1738 1739 /* Now clear reference and modify */ 1740 ref = pmap_clear_reference(pg); 1741 mod = pmap_clear_modify(pg); 1742 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1743 (void *)(u_long)va, (long)pa, 1744 ref, mod); 1745 1746 /* Modify page */ 1747 *loc = 1; 1748 1749 ref = pmap_is_referenced(pg); 1750 mod = pmap_is_modified(pg); 1751 printf("Modified page: ref %d, mod %d\n", 1752 ref, mod); 1753 1754 /* Now clear reference and modify */ 1755 ref = pmap_clear_reference(pg); 1756 mod = pmap_clear_modify(pg); 1757 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1758 (void *)(u_long)va, (long)pa, 1759 ref, mod); 1760 1761 /* Check it's properly cleared */ 1762 ref = pmap_is_referenced(pg); 1763 mod = pmap_is_modified(pg); 1764 printf("Checking cleared page: ref %d, mod %d\n", 1765 ref, mod); 1766 1767 /* Modify page */ 1768 *loc = 1; 1769 1770 ref = pmap_is_referenced(pg); 1771 mod = pmap_is_modified(pg); 1772 printf("Modified page: ref %d, mod %d\n", 1773 ref, mod); 1774 1775 /* Check pmap_protect() */ 1776 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_READ); 1777 pmap_update(pmap_kernel()); 1778 ref = pmap_is_referenced(pg); 1779 mod = pmap_is_modified(pg); 1780 printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n", 1781 ref, mod); 1782 1783 /* Now clear reference and modify */ 1784 ref = pmap_clear_reference(pg); 1785 mod = pmap_clear_modify(pg); 1786 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1787 (void *)(u_long)va, (long)pa, 1788 ref, mod); 1789 1790 /* Reference page */ 1791 val = *loc; 1792 1793 ref = pmap_is_referenced(pg); 1794 mod = pmap_is_modified(pg); 1795 printf("Referenced page: ref %d, mod %d val %x\n", 1796 ref, mod, val); 1797 1798 /* Now clear reference and modify */ 1799 ref = pmap_clear_reference(pg); 1800 mod = pmap_clear_modify(pg); 1801 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1802 (void *)(u_long)va, (long)pa, 1803 ref, mod); 1804 1805 /* Modify page */ 1806 #if 0 1807 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1808 pmap_update(pmap_kernel()); 1809 #endif 1810 *loc = 1; 1811 1812 ref = pmap_is_referenced(pg); 1813 mod = pmap_is_modified(pg); 1814 printf("Modified page: ref %d, mod %d\n", 1815 ref, mod); 1816 1817 /* Check pmap_protect() */ 1818 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_NONE); 1819 pmap_update(pmap_kernel()); 1820 ref = pmap_is_referenced(pg); 1821 mod = pmap_is_modified(pg); 1822 printf("pmap_protect(): ref %d, mod %d\n", 1823 ref, mod); 1824 1825 /* Now clear reference and modify */ 1826 ref = pmap_clear_reference(pg); 1827 mod = pmap_clear_modify(pg); 1828 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1829 (void *)(u_long)va, (long)pa, 1830 ref, mod); 1831 1832 /* Reference page */ 1833 val = *loc; 1834 1835 ref = pmap_is_referenced(pg); 1836 mod = pmap_is_modified(pg); 1837 printf("Referenced page: ref %d, mod %d val %x\n", 1838 ref, mod, val); 1839 1840 /* Now clear reference and modify */ 1841 ref = pmap_clear_reference(pg); 1842 mod = pmap_clear_modify(pg); 1843 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1844 (void *)(u_long)va, (long)pa, 1845 ref, mod); 1846 1847 /* Modify page */ 1848 #if 0 1849 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1850 pmap_update(pmap_kernel()); 1851 #endif 1852 *loc = 1; 1853 1854 ref = pmap_is_referenced(pg); 1855 mod = pmap_is_modified(pg); 1856 printf("Modified page: ref %d, mod %d\n", 1857 ref, mod); 1858 1859 /* Check pmap_pag_protect() */ 1860 pmap_page_protect(pg, VM_PROT_READ); 1861 ref = pmap_is_referenced(pg); 1862 mod = pmap_is_modified(pg); 1863 printf("pmap_page_protect(VM_PROT_READ): ref %d, mod %d\n", 1864 ref, mod); 1865 1866 /* Now clear reference and modify */ 1867 ref = pmap_clear_reference(pg); 1868 mod = pmap_clear_modify(pg); 1869 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1870 (void *)(u_long)va, (long)pa, 1871 ref, mod); 1872 1873 /* Reference page */ 1874 val = *loc; 1875 1876 ref = pmap_is_referenced(pg); 1877 mod = pmap_is_modified(pg); 1878 printf("Referenced page: ref %d, mod %d val %x\n", 1879 ref, mod, val); 1880 1881 /* Now clear reference and modify */ 1882 ref = pmap_clear_reference(pg); 1883 mod = pmap_clear_modify(pg); 1884 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1885 (void *)(u_long)va, (long)pa, 1886 ref, mod); 1887 1888 /* Modify page */ 1889 #if 0 1890 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1891 pmap_update(pmap_kernel()); 1892 #endif 1893 *loc = 1; 1894 1895 ref = pmap_is_referenced(pg); 1896 mod = pmap_is_modified(pg); 1897 printf("Modified page: ref %d, mod %d\n", 1898 ref, mod); 1899 1900 /* Check pmap_pag_protect() */ 1901 pmap_page_protect(pg, VM_PROT_NONE); 1902 ref = pmap_is_referenced(pg); 1903 mod = pmap_is_modified(pg); 1904 printf("pmap_page_protect(): ref %d, mod %d\n", 1905 ref, mod); 1906 1907 /* Now clear reference and modify */ 1908 ref = pmap_clear_reference(pg); 1909 mod = pmap_clear_modify(pg); 1910 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1911 (void *)(u_long)va, (long)pa, 1912 ref, mod); 1913 1914 1915 /* Reference page */ 1916 val = *loc; 1917 1918 ref = pmap_is_referenced(pg); 1919 mod = pmap_is_modified(pg); 1920 printf("Referenced page: ref %d, mod %d val %x\n", 1921 ref, mod, val); 1922 1923 /* Now clear reference and modify */ 1924 ref = pmap_clear_reference(pg); 1925 mod = pmap_clear_modify(pg); 1926 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1927 (void *)(u_long)va, (long)pa, 1928 ref, mod); 1929 1930 /* Modify page */ 1931 #if 0 1932 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 1933 pmap_update(pmap_kernel()); 1934 #endif 1935 *loc = 1; 1936 1937 ref = pmap_is_referenced(pg); 1938 mod = pmap_is_modified(pg); 1939 printf("Modified page: ref %d, mod %d\n", 1940 ref, mod); 1941 1942 /* Unmap page */ 1943 pmap_remove(pmap_kernel(), va, va+1); 1944 pmap_update(pmap_kernel()); 1945 ref = pmap_is_referenced(pg); 1946 mod = pmap_is_modified(pg); 1947 printf("Unmapped page: ref %d, mod %d\n", ref, mod); 1948 1949 /* Now clear reference and modify */ 1950 ref = pmap_clear_reference(pg); 1951 mod = pmap_clear_modify(pg); 1952 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 1953 (void *)(u_long)va, (long)pa, ref, mod); 1954 1955 /* Check it's properly cleared */ 1956 ref = pmap_is_referenced(pg); 1957 mod = pmap_is_modified(pg); 1958 printf("Checking cleared page: ref %d, mod %d\n", 1959 ref, mod); 1960 1961 pmap_remove(pmap_kernel(), va, va + PAGE_SIZE); 1962 pmap_kenter_pa(va, pa, VM_PROT_ALL, 0); 1963 uvm_km_free(kernel_map, (vaddr_t)va, PAGE_SIZE, UVM_KMF_WIRED); 1964 } 1965 #endif 1966