1 /* $NetBSD: booke_pmap.c,v 1.23 2015/11/05 06:26:15 pgoyette Exp $ */ 2 /*- 3 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects 8 * Agency and which was developed by Matt Thomas of 3am Software Foundry. 9 * 10 * This material is based upon work supported by the Defense Advanced Research 11 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under 12 * Contract No. N66001-09-C-2073. 13 * Approved for Public Release, Distribution Unlimited 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 26 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34 * POSSIBILITY OF SUCH DAMAGE. 35 */ 36 37 #define __PMAP_PRIVATE 38 39 #include <sys/cdefs.h> 40 41 __KERNEL_RCSID(0, "$NetBSD: booke_pmap.c,v 1.23 2015/11/05 06:26:15 pgoyette Exp $"); 42 43 #include <sys/param.h> 44 #include <sys/kcore.h> 45 #include <sys/buf.h> 46 #include <sys/mutex.h> 47 48 #include <uvm/uvm.h> 49 50 #include <machine/pmap.h> 51 52 #if defined(MULTIPROCESSOR) 53 kmutex_t pmap_tlb_miss_lock; 54 #endif 55 56 /* 57 * Initialize the kernel pmap. 58 */ 59 #ifdef MULTIPROCESSOR 60 #define PMAP_SIZE offsetof(struct pmap, pm_pai[PMAP_TLB_MAX]) 61 #else 62 #define PMAP_SIZE sizeof(struct pmap) 63 #endif 64 65 CTASSERT(sizeof(pmap_segtab_t) == NBPG); 66 67 pmap_segtab_t pmap_kernel_segtab; 68 69 void 70 pmap_procwr(struct proc *p, vaddr_t va, size_t len) 71 { 72 struct pmap * const pmap = p->p_vmspace->vm_map.pmap; 73 vsize_t off = va & PAGE_SIZE; 74 75 kpreempt_disable(); 76 for (const vaddr_t eva = va + len; va < eva; off = 0) { 77 const vaddr_t segeva = min(va + len, va - off + PAGE_SIZE); 78 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 79 if (ptep == NULL) { 80 va = segeva; 81 continue; 82 } 83 pt_entry_t pt_entry = *ptep; 84 if (!pte_valid_p(pt_entry) || !pte_exec_p(pt_entry)) { 85 va = segeva; 86 continue; 87 } 88 kpreempt_enable(); 89 dcache_wb(pte_to_paddr(pt_entry), segeva - va); 90 icache_inv(pte_to_paddr(pt_entry), segeva - va); 91 kpreempt_disable(); 92 va = segeva; 93 } 94 kpreempt_enable(); 95 } 96 97 void 98 pmap_md_page_syncicache(struct vm_page *pg, const kcpuset_t *onproc) 99 { 100 /* 101 * If onproc is empty, we could do a 102 * pmap_page_protect(pg, VM_PROT_NONE) and remove all 103 * mappings of the page and clear its execness. Then 104 * the next time page is faulted, it will get icache 105 * synched. But this is easier. :) 106 */ 107 paddr_t pa = VM_PAGE_TO_PHYS(pg); 108 dcache_wb_page(pa); 109 icache_inv_page(pa); 110 } 111 112 vaddr_t 113 pmap_md_direct_map_paddr(paddr_t pa) 114 { 115 return (vaddr_t) pa; 116 } 117 118 bool 119 pmap_md_direct_mapped_vaddr_p(vaddr_t va) 120 { 121 return va < VM_MIN_KERNEL_ADDRESS || VM_MAX_KERNEL_ADDRESS <= va; 122 } 123 124 paddr_t 125 pmap_md_direct_mapped_vaddr_to_paddr(vaddr_t va) 126 { 127 return (paddr_t) va; 128 } 129 130 #ifdef PMAP_MINIMALTLB 131 static pt_entry_t * 132 kvtopte(const pmap_segtab_t *stp, vaddr_t va) 133 { 134 pt_entry_t * const ptep = stp->seg_tab[va >> SEGSHIFT]; 135 if (ptep == NULL) 136 return NULL; 137 return &ptep[(va & SEGOFSET) >> PAGE_SHIFT]; 138 } 139 140 vaddr_t 141 pmap_kvptefill(vaddr_t sva, vaddr_t eva, pt_entry_t pt_entry) 142 { 143 const pmap_segtab_t * const stp = pmap_kernel()->pm_segtab; 144 KASSERT(sva == trunc_page(sva)); 145 pt_entry_t *ptep = kvtopte(stp, sva); 146 for (; sva < eva; sva += NBPG) { 147 *ptep++ = pt_entry ? (sva | pt_entry) : 0; 148 } 149 return sva; 150 } 151 #endif 152 153 /* 154 * Bootstrap the system enough to run with virtual memory. 155 * firstaddr is the first unused kseg0 address (not page aligned). 156 */ 157 vaddr_t 158 pmap_bootstrap(vaddr_t startkernel, vaddr_t endkernel, 159 phys_ram_seg_t *avail, size_t cnt) 160 { 161 pmap_segtab_t * const stp = &pmap_kernel_segtab; 162 163 /* 164 * Initialize the kernel segment table. 165 */ 166 pmap_kernel()->pm_segtab = stp; 167 curcpu()->ci_pmap_kern_segtab = stp; 168 169 KASSERT(endkernel == trunc_page(endkernel)); 170 171 /* init the lock */ 172 pmap_tlb_info_init(&pmap_tlb0_info); 173 174 #if defined(MULTIPROCESSOR) 175 mutex_init(&pmap_tlb_miss_lock, MUTEX_SPIN, IPL_HIGH); 176 #endif 177 178 /* 179 * Compute the number of pages kmem_arena will have. 180 */ 181 kmeminit_nkmempages(); 182 183 /* 184 * Figure out how many PTE's are necessary to map the kernel. 185 * We also reserve space for kmem_alloc_pageable() for vm_fork(). 186 */ 187 188 /* Get size of buffer cache and set an upper limit */ 189 buf_setvalimit((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / 8); 190 vsize_t bufsz = buf_memcalc(); 191 buf_setvalimit(bufsz); 192 193 vsize_t kv_nsegtabs = pmap_round_seg(VM_PHYS_SIZE 194 + (ubc_nwins << ubc_winshift) 195 + bufsz 196 + 16 * NCARGS 197 + pager_map_size 198 + maxproc * USPACE 199 + NBPG * nkmempages) >> SEGSHIFT; 200 201 /* 202 * Initialize `FYI' variables. Note we're relying on 203 * the fact that BSEARCH sorts the vm_physmem[] array 204 * for us. Must do this before uvm_pageboot_alloc() 205 * can be called. 206 */ 207 pmap_limits.avail_start = vm_physmem[0].start << PGSHIFT; 208 pmap_limits.avail_end = vm_physmem[vm_nphysseg - 1].end << PGSHIFT; 209 const size_t max_nsegtabs = 210 (pmap_round_seg(VM_MAX_KERNEL_ADDRESS) 211 - pmap_trunc_seg(VM_MIN_KERNEL_ADDRESS)) / NBSEG; 212 if (kv_nsegtabs >= max_nsegtabs) { 213 pmap_limits.virtual_end = VM_MAX_KERNEL_ADDRESS; 214 kv_nsegtabs = max_nsegtabs; 215 } else { 216 pmap_limits.virtual_end = VM_MIN_KERNEL_ADDRESS 217 + kv_nsegtabs * NBSEG; 218 } 219 220 /* 221 * Now actually allocate the kernel PTE array (must be done 222 * after virtual_end is initialized). 223 */ 224 const vaddr_t kv_segtabs = avail[0].start; 225 KASSERT(kv_segtabs == endkernel); 226 KASSERT(avail[0].size >= NBPG * kv_nsegtabs); 227 printf(" kv_nsegtabs=%#"PRIxVSIZE, kv_nsegtabs); 228 printf(" kv_segtabs=%#"PRIxVADDR, kv_segtabs); 229 avail[0].start += NBPG * kv_nsegtabs; 230 avail[0].size -= NBPG * kv_nsegtabs; 231 endkernel += NBPG * kv_nsegtabs; 232 233 /* 234 * Initialize the kernel's two-level page level. This only wastes 235 * an extra page for the segment table and allows the user/kernel 236 * access to be common. 237 */ 238 pt_entry_t **ptp = &stp->seg_tab[VM_MIN_KERNEL_ADDRESS >> SEGSHIFT]; 239 pt_entry_t *ptep = (void *)kv_segtabs; 240 memset(ptep, 0, NBPG * kv_nsegtabs); 241 for (size_t i = 0; i < kv_nsegtabs; i++, ptep += NPTEPG) { 242 *ptp++ = ptep; 243 } 244 245 #if PMAP_MINIMALTLB 246 const vsize_t dm_nsegtabs = (physmem + NPTEPG - 1) / NPTEPG; 247 const vaddr_t dm_segtabs = avail[0].start; 248 printf(" dm_nsegtabs=%#"PRIxVSIZE, dm_nsegtabs); 249 printf(" dm_segtabs=%#"PRIxVADDR, dm_segtabs); 250 KASSERT(dm_segtabs == endkernel); 251 KASSERT(avail[0].size >= NBPG * dm_nsegtabs); 252 avail[0].start += NBPG * dm_nsegtabs; 253 avail[0].size -= NBPG * dm_nsegtabs; 254 endkernel += NBPG * dm_nsegtabs; 255 256 ptp = stp->seg_tab; 257 ptep = (void *)dm_segtabs; 258 memset(ptep, 0, NBPG * dm_nsegtabs); 259 for (size_t i = 0; i < dm_nsegtabs; i++, ptp++, ptep += NPTEPG) { 260 *ptp = ptep; 261 } 262 263 /* 264 */ 265 extern uint32_t _fdata[], _etext[]; 266 vaddr_t va; 267 268 /* Now make everything before the kernel inaccessible. */ 269 va = pmap_kvptefill(NBPG, startkernel, 0); 270 271 /* Kernel text is readonly & executable */ 272 va = pmap_kvptefill(va, round_page((vaddr_t)_etext), 273 PTE_M | PTE_xR | PTE_xX); 274 275 /* Kernel .rdata is readonly */ 276 va = pmap_kvptefill(va, trunc_page((vaddr_t)_fdata), PTE_M | PTE_xR); 277 278 /* Kernel .data/.bss + page tables are read-write */ 279 va = pmap_kvptefill(va, round_page(endkernel), PTE_M | PTE_xR | PTE_xW); 280 281 /* message buffer page table pages are read-write */ 282 (void) pmap_kvptefill(msgbuf_paddr, msgbuf_paddr+round_page(MSGBUFSIZE), 283 PTE_M | PTE_xR | PTE_xW); 284 #endif 285 286 for (size_t i = 0; i < cnt; i++) { 287 printf(" uvm_page_physload(%#lx,%#lx,%#lx,%#lx,%d)", 288 atop(avail[i].start), 289 atop(avail[i].start + avail[i].size) - 1, 290 atop(avail[i].start), 291 atop(avail[i].start + avail[i].size) - 1, 292 VM_FREELIST_DEFAULT); 293 uvm_page_physload( 294 atop(avail[i].start), 295 atop(avail[i].start + avail[i].size) - 1, 296 atop(avail[i].start), 297 atop(avail[i].start + avail[i].size) - 1, 298 VM_FREELIST_DEFAULT); 299 } 300 301 pmap_pvlist_lock_init(curcpu()->ci_ci.dcache_line_size); 302 303 /* 304 * Initialize the pools. 305 */ 306 pool_init(&pmap_pmap_pool, PMAP_SIZE, 0, 0, 0, "pmappl", 307 &pool_allocator_nointr, IPL_NONE); 308 pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvpl", 309 &pmap_pv_page_allocator, IPL_NONE); 310 311 tlb_set_asid(0); 312 313 return endkernel; 314 } 315 316 struct vm_page * 317 pmap_md_alloc_poolpage(int flags) 318 { 319 /* 320 * Any managed page works for us. 321 */ 322 return uvm_pagealloc(NULL, 0, NULL, flags); 323 } 324 325 vaddr_t 326 pmap_md_map_poolpage(paddr_t pa, vsize_t size) 327 { 328 const vaddr_t sva = (vaddr_t) pa; 329 #ifdef PMAP_MINIMALTLB 330 const vaddr_t eva = sva + size; 331 pmap_kvptefill(sva, eva, PTE_M | PTE_xR | PTE_xW); 332 #endif 333 return sva; 334 } 335 336 void 337 pmap_md_unmap_poolpage(vaddr_t va, vsize_t size) 338 { 339 #ifdef PMAP_MINIMALTLB 340 struct pmap * const pm = pmap_kernel(); 341 const vaddr_t eva = va + size; 342 pmap_kvptefill(va, eva, 0); 343 for (;va < eva; va += NBPG) { 344 pmap_tlb_invalidate_addr(pm, va); 345 } 346 pmap_update(pm); 347 #endif 348 } 349 350 void 351 pmap_zero_page(paddr_t pa) 352 { 353 vaddr_t va = pmap_md_map_poolpage(pa, NBPG); 354 dcache_zero_page(va); 355 356 KASSERT(!VM_PAGEMD_EXECPAGE_P(VM_PAGE_TO_MD(PHYS_TO_VM_PAGE(va)))); 357 pmap_md_unmap_poolpage(va, NBPG); 358 } 359 360 void 361 pmap_copy_page(paddr_t src, paddr_t dst) 362 { 363 const size_t line_size = curcpu()->ci_ci.dcache_line_size; 364 vaddr_t src_va = pmap_md_map_poolpage(src, NBPG); 365 vaddr_t dst_va = pmap_md_map_poolpage(dst, NBPG); 366 const vaddr_t end = src_va + PAGE_SIZE; 367 368 while (src_va < end) { 369 __asm __volatile( 370 "dcbt %2,%0" "\n\t" /* touch next src cacheline */ 371 "dcba 0,%1" "\n\t" /* don't fetch dst cacheline */ 372 :: "b"(src_va), "b"(dst_va), "b"(line_size)); 373 for (u_int i = 0; 374 i < line_size; 375 src_va += 32, dst_va += 32, i += 32) { 376 register_t tmp; 377 __asm __volatile( 378 "mr %[tmp],31" "\n\t" 379 "lmw 24,0(%[src])" "\n\t" 380 "stmw 24,0(%[dst])" "\n\t" 381 "mr 31,%[tmp]" "\n\t" 382 : [tmp] "=&r"(tmp) 383 : [src] "b"(src_va), [dst] "b"(dst_va) 384 : "r24", "r25", "r26", "r27", 385 "r28", "r29", "r30", "memory"); 386 } 387 } 388 pmap_md_unmap_poolpage(src_va, NBPG); 389 pmap_md_unmap_poolpage(dst_va, NBPG); 390 391 KASSERT(!VM_PAGEMD_EXECPAGE_P(VM_PAGE_TO_MD(PHYS_TO_VM_PAGE(dst)))); 392 } 393 394 void 395 pmap_md_init(void) 396 { 397 398 /* nothing for now */ 399 } 400 401 bool 402 pmap_md_io_vaddr_p(vaddr_t va) 403 { 404 return va >= pmap_limits.avail_end 405 && !(VM_MIN_KERNEL_ADDRESS <= va && va < VM_MAX_KERNEL_ADDRESS); 406 } 407 408 bool 409 pmap_md_tlb_check_entry(void *ctx, vaddr_t va, tlb_asid_t asid, pt_entry_t pte) 410 { 411 pmap_t pm = ctx; 412 struct pmap_asid_info * const pai = PMAP_PAI(pm, curcpu()->ci_tlb_info); 413 414 if (asid != pai->pai_asid) 415 return true; 416 417 const pt_entry_t * const ptep = pmap_pte_lookup(pm, va); 418 KASSERT(ptep != NULL); 419 pt_entry_t xpte = *ptep; 420 xpte &= ~((xpte & (PTE_UNSYNCED|PTE_UNMODIFIED)) << 1); 421 xpte ^= xpte & (PTE_UNSYNCED|PTE_UNMODIFIED|PTE_WIRED); 422 423 KASSERTMSG(pte == xpte, 424 "pm=%p va=%#"PRIxVADDR" asid=%u: TLB pte (%#x) != real pte (%#x/%#x)", 425 pm, va, asid, pte, xpte, *ptep); 426 427 return true; 428 } 429 430 #ifdef MULTIPROCESSOR 431 void 432 pmap_md_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci) 433 { 434 /* nothing */ 435 } 436 437 void 438 pmap_md_tlb_miss_lock_enter(void) 439 { 440 441 mutex_spin_enter(&pmap_tlb_miss_lock); 442 } 443 444 void 445 pmap_md_tlb_miss_lock_exit(void) 446 { 447 448 mutex_spin_exit(&pmap_tlb_miss_lock); 449 } 450 #endif /* MULTIPROCESSOR */ 451