1*0Sstevel@tonic-gate /* 2*0Sstevel@tonic-gate * CDDL HEADER START 3*0Sstevel@tonic-gate * 4*0Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*0Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*0Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*0Sstevel@tonic-gate * with the License. 8*0Sstevel@tonic-gate * 9*0Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*0Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*0Sstevel@tonic-gate * See the License for the specific language governing permissions 12*0Sstevel@tonic-gate * and limitations under the License. 13*0Sstevel@tonic-gate * 14*0Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*0Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*0Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*0Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*0Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*0Sstevel@tonic-gate * 20*0Sstevel@tonic-gate * CDDL HEADER END 21*0Sstevel@tonic-gate */ 22*0Sstevel@tonic-gate /* 23*0Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24*0Sstevel@tonic-gate * Use is subject to license terms. 25*0Sstevel@tonic-gate */ 26*0Sstevel@tonic-gate 27*0Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 28*0Sstevel@tonic-gate 29*0Sstevel@tonic-gate #include <sys/types.h> 30*0Sstevel@tonic-gate #include <sys/systm.h> 31*0Sstevel@tonic-gate #include <sys/archsystm.h> 32*0Sstevel@tonic-gate #include <sys/machsystm.h> 33*0Sstevel@tonic-gate #include <sys/t_lock.h> 34*0Sstevel@tonic-gate #include <sys/vmem.h> 35*0Sstevel@tonic-gate #include <sys/mman.h> 36*0Sstevel@tonic-gate #include <sys/vm.h> 37*0Sstevel@tonic-gate #include <sys/cpu.h> 38*0Sstevel@tonic-gate #include <sys/cmn_err.h> 39*0Sstevel@tonic-gate #include <sys/cpuvar.h> 40*0Sstevel@tonic-gate #include <sys/atomic.h> 41*0Sstevel@tonic-gate #include <vm/as.h> 42*0Sstevel@tonic-gate #include <vm/hat.h> 43*0Sstevel@tonic-gate #include <vm/as.h> 44*0Sstevel@tonic-gate #include <vm/page.h> 45*0Sstevel@tonic-gate #include <vm/seg.h> 46*0Sstevel@tonic-gate #include <vm/seg_kmem.h> 47*0Sstevel@tonic-gate #include <vm/hat_sfmmu.h> 48*0Sstevel@tonic-gate #include <sys/debug.h> 49*0Sstevel@tonic-gate #include <sys/cpu_module.h> 50*0Sstevel@tonic-gate 51*0Sstevel@tonic-gate /* 52*0Sstevel@tonic-gate * A quick way to generate a cache consistent address to map in a page. 53*0Sstevel@tonic-gate * users: ppcopy, pagezero, /proc, dev/mem 54*0Sstevel@tonic-gate * 55*0Sstevel@tonic-gate * The ppmapin/ppmapout routines provide a quick way of generating a cache 56*0Sstevel@tonic-gate * consistent address by reserving a given amount of kernel address space. 57*0Sstevel@tonic-gate * The base is PPMAPBASE and its size is PPMAPSIZE. This memory is divided 58*0Sstevel@tonic-gate * into x number of sets, where x is the number of colors for the virtual 59*0Sstevel@tonic-gate * cache. The number of colors is how many times a page can be mapped 60*0Sstevel@tonic-gate * simulatenously in the cache. For direct map caches this translates to 61*0Sstevel@tonic-gate * the number of pages in the cache. 62*0Sstevel@tonic-gate * Each set will be assigned a group of virtual pages from the reserved memory 63*0Sstevel@tonic-gate * depending on its virtual color. 64*0Sstevel@tonic-gate * When trying to assign a virtual address we will find out the color for the 65*0Sstevel@tonic-gate * physical page in question (if applicable). Then we will try to find an 66*0Sstevel@tonic-gate * available virtual page from the set of the appropiate color. 67*0Sstevel@tonic-gate */ 68*0Sstevel@tonic-gate 69*0Sstevel@tonic-gate #define clsettoarray(color, set) ((color * nsets) + set) 70*0Sstevel@tonic-gate 71*0Sstevel@tonic-gate int pp_slots = 4; /* small default, tuned by cpu module */ 72*0Sstevel@tonic-gate 73*0Sstevel@tonic-gate /* tuned by cpu module, default is "safe" */ 74*0Sstevel@tonic-gate int pp_consistent_coloring = PPAGE_STORES_POLLUTE | PPAGE_LOADS_POLLUTE; 75*0Sstevel@tonic-gate 76*0Sstevel@tonic-gate static caddr_t ppmap_vaddrs[PPMAPSIZE / MMU_PAGESIZE]; 77*0Sstevel@tonic-gate static int nsets; /* number of sets */ 78*0Sstevel@tonic-gate static int ppmap_pages; /* generate align mask */ 79*0Sstevel@tonic-gate static int ppmap_shift; /* set selector */ 80*0Sstevel@tonic-gate 81*0Sstevel@tonic-gate #ifdef PPDEBUG 82*0Sstevel@tonic-gate #define MAXCOLORS 16 /* for debug only */ 83*0Sstevel@tonic-gate static int ppalloc_noslot = 0; /* # of allocations from kernelmap */ 84*0Sstevel@tonic-gate static int align_hits[MAXCOLORS]; 85*0Sstevel@tonic-gate static int pp_allocs; /* # of ppmapin requests */ 86*0Sstevel@tonic-gate #endif /* PPDEBUG */ 87*0Sstevel@tonic-gate 88*0Sstevel@tonic-gate /* 89*0Sstevel@tonic-gate * There are only 64 TLB entries on spitfire, 16 on cheetah 90*0Sstevel@tonic-gate * (fully-associative TLB) so we allow the cpu module to tune the 91*0Sstevel@tonic-gate * number to use here via pp_slots. 92*0Sstevel@tonic-gate */ 93*0Sstevel@tonic-gate static struct ppmap_va { 94*0Sstevel@tonic-gate caddr_t ppmap_slots[MAXPP_SLOTS]; 95*0Sstevel@tonic-gate } ppmap_va[NCPU]; 96*0Sstevel@tonic-gate 97*0Sstevel@tonic-gate void 98*0Sstevel@tonic-gate ppmapinit(void) 99*0Sstevel@tonic-gate { 100*0Sstevel@tonic-gate int color, nset, setsize; 101*0Sstevel@tonic-gate caddr_t va; 102*0Sstevel@tonic-gate 103*0Sstevel@tonic-gate ASSERT(pp_slots <= MAXPP_SLOTS); 104*0Sstevel@tonic-gate 105*0Sstevel@tonic-gate va = (caddr_t)PPMAPBASE; 106*0Sstevel@tonic-gate if (cache & CACHE_VAC) { 107*0Sstevel@tonic-gate int a; 108*0Sstevel@tonic-gate 109*0Sstevel@tonic-gate ppmap_pages = mmu_btop(shm_alignment); 110*0Sstevel@tonic-gate nsets = PPMAPSIZE / shm_alignment; 111*0Sstevel@tonic-gate setsize = shm_alignment; 112*0Sstevel@tonic-gate ppmap_shift = MMU_PAGESHIFT; 113*0Sstevel@tonic-gate a = ppmap_pages; 114*0Sstevel@tonic-gate while (a >>= 1) 115*0Sstevel@tonic-gate ppmap_shift++; 116*0Sstevel@tonic-gate } else { 117*0Sstevel@tonic-gate /* 118*0Sstevel@tonic-gate * If we do not have a virtual indexed cache we simply 119*0Sstevel@tonic-gate * have only one set containing all pages. 120*0Sstevel@tonic-gate */ 121*0Sstevel@tonic-gate ppmap_pages = 1; 122*0Sstevel@tonic-gate nsets = mmu_btop(PPMAPSIZE); 123*0Sstevel@tonic-gate setsize = MMU_PAGESIZE; 124*0Sstevel@tonic-gate ppmap_shift = MMU_PAGESHIFT; 125*0Sstevel@tonic-gate } 126*0Sstevel@tonic-gate for (color = 0; color < ppmap_pages; color++) { 127*0Sstevel@tonic-gate for (nset = 0; nset < nsets; nset++) { 128*0Sstevel@tonic-gate ppmap_vaddrs[clsettoarray(color, nset)] = 129*0Sstevel@tonic-gate (caddr_t)((uintptr_t)va + (nset * setsize)); 130*0Sstevel@tonic-gate } 131*0Sstevel@tonic-gate va += MMU_PAGESIZE; 132*0Sstevel@tonic-gate } 133*0Sstevel@tonic-gate } 134*0Sstevel@tonic-gate 135*0Sstevel@tonic-gate /* 136*0Sstevel@tonic-gate * Allocate a cache consistent virtual address to map a page, pp, 137*0Sstevel@tonic-gate * with protection, vprot; and map it in the MMU, using the most 138*0Sstevel@tonic-gate * efficient means possible. The argument avoid is a virtual address 139*0Sstevel@tonic-gate * hint which when masked yields an offset into a virtual cache 140*0Sstevel@tonic-gate * that should be avoided when allocating an address to map in a 141*0Sstevel@tonic-gate * page. An avoid arg of -1 means you don't care, for instance pagezero. 142*0Sstevel@tonic-gate * 143*0Sstevel@tonic-gate * machine dependent, depends on virtual address space layout, 144*0Sstevel@tonic-gate * understands that all kernel addresses have bit 31 set. 145*0Sstevel@tonic-gate * 146*0Sstevel@tonic-gate * NOTE: For sun4 platforms the meaning of the hint argument is opposite from 147*0Sstevel@tonic-gate * that found in other architectures. In other architectures the hint 148*0Sstevel@tonic-gate * (called avoid) was used to ask ppmapin to NOT use the specified cache color. 149*0Sstevel@tonic-gate * This was used to avoid virtual cache trashing in the bcopy. Unfortunately 150*0Sstevel@tonic-gate * in the case of a COW, this later on caused a cache aliasing conflict. In 151*0Sstevel@tonic-gate * sun4, the bcopy routine uses the block ld/st instructions so we don't have 152*0Sstevel@tonic-gate * to worry about virtual cache trashing. Actually, by using the hint to choose 153*0Sstevel@tonic-gate * the right color we can almost guarantee a cache conflict will not occur. 154*0Sstevel@tonic-gate */ 155*0Sstevel@tonic-gate 156*0Sstevel@tonic-gate caddr_t 157*0Sstevel@tonic-gate ppmapin(page_t *pp, uint_t vprot, caddr_t hint) 158*0Sstevel@tonic-gate { 159*0Sstevel@tonic-gate int color, nset, index, start; 160*0Sstevel@tonic-gate caddr_t va; 161*0Sstevel@tonic-gate 162*0Sstevel@tonic-gate #ifdef PPDEBUG 163*0Sstevel@tonic-gate pp_allocs++; 164*0Sstevel@tonic-gate #endif /* PPDEBUG */ 165*0Sstevel@tonic-gate if (cache & CACHE_VAC) { 166*0Sstevel@tonic-gate color = sfmmu_get_ppvcolor(pp); 167*0Sstevel@tonic-gate if (color == -1) { 168*0Sstevel@tonic-gate if ((intptr_t)hint != -1L) { 169*0Sstevel@tonic-gate color = addr_to_vcolor(hint); 170*0Sstevel@tonic-gate } else { 171*0Sstevel@tonic-gate color = addr_to_vcolor(mmu_ptob(pp->p_pagenum)); 172*0Sstevel@tonic-gate } 173*0Sstevel@tonic-gate } 174*0Sstevel@tonic-gate 175*0Sstevel@tonic-gate } else { 176*0Sstevel@tonic-gate /* 177*0Sstevel@tonic-gate * For physical caches, we can pick any address we want. 178*0Sstevel@tonic-gate */ 179*0Sstevel@tonic-gate color = 0; 180*0Sstevel@tonic-gate } 181*0Sstevel@tonic-gate 182*0Sstevel@tonic-gate start = color; 183*0Sstevel@tonic-gate do { 184*0Sstevel@tonic-gate for (nset = 0; nset < nsets; nset++) { 185*0Sstevel@tonic-gate index = clsettoarray(color, nset); 186*0Sstevel@tonic-gate va = ppmap_vaddrs[index]; 187*0Sstevel@tonic-gate if (va != NULL) { 188*0Sstevel@tonic-gate #ifdef PPDEBUG 189*0Sstevel@tonic-gate align_hits[color]++; 190*0Sstevel@tonic-gate #endif /* PPDEBUG */ 191*0Sstevel@tonic-gate if (casptr(&ppmap_vaddrs[index], 192*0Sstevel@tonic-gate va, NULL) == va) { 193*0Sstevel@tonic-gate hat_memload(kas.a_hat, va, pp, 194*0Sstevel@tonic-gate vprot | HAT_NOSYNC, 195*0Sstevel@tonic-gate HAT_LOAD_LOCK); 196*0Sstevel@tonic-gate return (va); 197*0Sstevel@tonic-gate } 198*0Sstevel@tonic-gate } 199*0Sstevel@tonic-gate } 200*0Sstevel@tonic-gate /* 201*0Sstevel@tonic-gate * first pick didn't succeed, try another 202*0Sstevel@tonic-gate */ 203*0Sstevel@tonic-gate if (++color == ppmap_pages) 204*0Sstevel@tonic-gate color = 0; 205*0Sstevel@tonic-gate } while (color != start); 206*0Sstevel@tonic-gate 207*0Sstevel@tonic-gate #ifdef PPDEBUG 208*0Sstevel@tonic-gate ppalloc_noslot++; 209*0Sstevel@tonic-gate #endif /* PPDEBUG */ 210*0Sstevel@tonic-gate 211*0Sstevel@tonic-gate /* 212*0Sstevel@tonic-gate * No free slots; get a random one from the kernel heap area. 213*0Sstevel@tonic-gate */ 214*0Sstevel@tonic-gate va = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP); 215*0Sstevel@tonic-gate 216*0Sstevel@tonic-gate hat_memload(kas.a_hat, va, pp, vprot | HAT_NOSYNC, HAT_LOAD_LOCK); 217*0Sstevel@tonic-gate 218*0Sstevel@tonic-gate return (va); 219*0Sstevel@tonic-gate 220*0Sstevel@tonic-gate } 221*0Sstevel@tonic-gate 222*0Sstevel@tonic-gate void 223*0Sstevel@tonic-gate ppmapout(caddr_t va) 224*0Sstevel@tonic-gate { 225*0Sstevel@tonic-gate int color, nset, index; 226*0Sstevel@tonic-gate 227*0Sstevel@tonic-gate if (va >= kernelheap && va < ekernelheap) { 228*0Sstevel@tonic-gate /* 229*0Sstevel@tonic-gate * Space came from kernelmap, flush the page and 230*0Sstevel@tonic-gate * return the space. 231*0Sstevel@tonic-gate */ 232*0Sstevel@tonic-gate hat_unload(kas.a_hat, va, PAGESIZE, 233*0Sstevel@tonic-gate (HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK)); 234*0Sstevel@tonic-gate vmem_free(heap_arena, va, PAGESIZE); 235*0Sstevel@tonic-gate } else { 236*0Sstevel@tonic-gate /* 237*0Sstevel@tonic-gate * Space came from ppmap_vaddrs[], give it back. 238*0Sstevel@tonic-gate */ 239*0Sstevel@tonic-gate color = addr_to_vcolor(va); 240*0Sstevel@tonic-gate ASSERT((cache & CACHE_VAC)? (color < ppmap_pages) : 1); 241*0Sstevel@tonic-gate 242*0Sstevel@tonic-gate nset = ((uintptr_t)va >> ppmap_shift) & (nsets - 1); 243*0Sstevel@tonic-gate index = clsettoarray(color, nset); 244*0Sstevel@tonic-gate hat_unload(kas.a_hat, va, PAGESIZE, 245*0Sstevel@tonic-gate (HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK)); 246*0Sstevel@tonic-gate 247*0Sstevel@tonic-gate ASSERT(ppmap_vaddrs[index] == NULL); 248*0Sstevel@tonic-gate ppmap_vaddrs[index] = va; 249*0Sstevel@tonic-gate } 250*0Sstevel@tonic-gate } 251*0Sstevel@tonic-gate 252*0Sstevel@tonic-gate #ifdef DEBUG 253*0Sstevel@tonic-gate #define PP_STAT_ADD(stat) (stat)++ 254*0Sstevel@tonic-gate uint_t pload, ploadfail; 255*0Sstevel@tonic-gate uint_t ppzero, ppzero_short; 256*0Sstevel@tonic-gate #else 257*0Sstevel@tonic-gate #define PP_STAT_ADD(stat) 258*0Sstevel@tonic-gate #endif /* DEBUG */ 259*0Sstevel@tonic-gate 260*0Sstevel@tonic-gate static void 261*0Sstevel@tonic-gate pp_unload_tlb(caddr_t *pslot, caddr_t va) 262*0Sstevel@tonic-gate { 263*0Sstevel@tonic-gate ASSERT(*pslot == va); 264*0Sstevel@tonic-gate 265*0Sstevel@tonic-gate vtag_flushpage(va, KCONTEXT); 266*0Sstevel@tonic-gate *pslot = NULL; /* release the slot */ 267*0Sstevel@tonic-gate } 268*0Sstevel@tonic-gate 269*0Sstevel@tonic-gate /* 270*0Sstevel@tonic-gate * Routine to copy kernel pages during relocation. It will copy one 271*0Sstevel@tonic-gate * PAGESIZE page to another PAGESIZE page. This function may be called 272*0Sstevel@tonic-gate * above LOCK_LEVEL so it should not grab any locks. 273*0Sstevel@tonic-gate */ 274*0Sstevel@tonic-gate void 275*0Sstevel@tonic-gate ppcopy_kernel__relocatable(page_t *fm_pp, page_t *to_pp) 276*0Sstevel@tonic-gate { 277*0Sstevel@tonic-gate uint64_t fm_pa, to_pa; 278*0Sstevel@tonic-gate size_t nbytes; 279*0Sstevel@tonic-gate 280*0Sstevel@tonic-gate fm_pa = (uint64_t)(fm_pp->p_pagenum) << MMU_PAGESHIFT; 281*0Sstevel@tonic-gate to_pa = (uint64_t)(to_pp->p_pagenum) << MMU_PAGESHIFT; 282*0Sstevel@tonic-gate 283*0Sstevel@tonic-gate nbytes = MMU_PAGESIZE; 284*0Sstevel@tonic-gate 285*0Sstevel@tonic-gate for (; nbytes > 0; fm_pa += 32, to_pa += 32, nbytes -= 32) 286*0Sstevel@tonic-gate hw_pa_bcopy32(fm_pa, to_pa); 287*0Sstevel@tonic-gate } 288*0Sstevel@tonic-gate 289*0Sstevel@tonic-gate /* 290*0Sstevel@tonic-gate * Copy the data from the physical page represented by "frompp" to 291*0Sstevel@tonic-gate * that represented by "topp". 292*0Sstevel@tonic-gate * 293*0Sstevel@tonic-gate * Try to use per cpu mapping first, if that fails then call pp_mapin 294*0Sstevel@tonic-gate * to load it. 295*0Sstevel@tonic-gate */ 296*0Sstevel@tonic-gate void 297*0Sstevel@tonic-gate ppcopy(page_t *fm_pp, page_t *to_pp) 298*0Sstevel@tonic-gate { 299*0Sstevel@tonic-gate caddr_t fm_va, to_va; 300*0Sstevel@tonic-gate 301*0Sstevel@tonic-gate fm_va = ppmapin(fm_pp, PROT_READ, (caddr_t)-1); 302*0Sstevel@tonic-gate to_va = ppmapin(to_pp, PROT_READ | PROT_WRITE, fm_va); 303*0Sstevel@tonic-gate bcopy(fm_va, to_va, PAGESIZE); 304*0Sstevel@tonic-gate ppmapout(fm_va); 305*0Sstevel@tonic-gate ppmapout(to_va); 306*0Sstevel@tonic-gate } 307*0Sstevel@tonic-gate 308*0Sstevel@tonic-gate /* 309*0Sstevel@tonic-gate * Zero the physical page from off to off + len given by `pp' 310*0Sstevel@tonic-gate * without changing the reference and modified bits of page. 311*0Sstevel@tonic-gate * 312*0Sstevel@tonic-gate * Again, we'll try per cpu mapping first. 313*0Sstevel@tonic-gate */ 314*0Sstevel@tonic-gate void 315*0Sstevel@tonic-gate pagezero(page_t *pp, uint_t off, uint_t len) 316*0Sstevel@tonic-gate { 317*0Sstevel@tonic-gate caddr_t va; 318*0Sstevel@tonic-gate extern int hwblkclr(void *, size_t); 319*0Sstevel@tonic-gate extern int use_hw_bzero; 320*0Sstevel@tonic-gate 321*0Sstevel@tonic-gate ASSERT((int)len > 0 && (int)off >= 0 && off + len <= PAGESIZE); 322*0Sstevel@tonic-gate ASSERT(PAGE_LOCKED(pp)); 323*0Sstevel@tonic-gate 324*0Sstevel@tonic-gate PP_STAT_ADD(ppzero); 325*0Sstevel@tonic-gate 326*0Sstevel@tonic-gate if (len != MMU_PAGESIZE || !use_hw_bzero) { 327*0Sstevel@tonic-gate PP_STAT_ADD(ppzero_short); 328*0Sstevel@tonic-gate } 329*0Sstevel@tonic-gate 330*0Sstevel@tonic-gate kpreempt_disable(); 331*0Sstevel@tonic-gate 332*0Sstevel@tonic-gate va = ppmapin(pp, PROT_READ | PROT_WRITE, (caddr_t)-1); 333*0Sstevel@tonic-gate 334*0Sstevel@tonic-gate if (!use_hw_bzero) { 335*0Sstevel@tonic-gate bzero(va + off, len); 336*0Sstevel@tonic-gate sync_icache(va + off, len); 337*0Sstevel@tonic-gate } else if (hwblkclr(va + off, len)) { 338*0Sstevel@tonic-gate /* 339*0Sstevel@tonic-gate * We may not have used block commit asi. 340*0Sstevel@tonic-gate * So flush the I-$ manually 341*0Sstevel@tonic-gate */ 342*0Sstevel@tonic-gate sync_icache(va + off, len); 343*0Sstevel@tonic-gate } else { 344*0Sstevel@tonic-gate /* 345*0Sstevel@tonic-gate * We have used blk commit, and flushed the I-$. However we 346*0Sstevel@tonic-gate * still may have an instruction in the pipeline. Only a flush 347*0Sstevel@tonic-gate * instruction will invalidate that. 348*0Sstevel@tonic-gate */ 349*0Sstevel@tonic-gate doflush(va); 350*0Sstevel@tonic-gate } 351*0Sstevel@tonic-gate 352*0Sstevel@tonic-gate ppmapout(va); 353*0Sstevel@tonic-gate kpreempt_enable(); 354*0Sstevel@tonic-gate } 355