1*0Sstevel@tonic-gate /* 2*0Sstevel@tonic-gate * CDDL HEADER START 3*0Sstevel@tonic-gate * 4*0Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*0Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*0Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*0Sstevel@tonic-gate * with the License. 8*0Sstevel@tonic-gate * 9*0Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*0Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*0Sstevel@tonic-gate * See the License for the specific language governing permissions 12*0Sstevel@tonic-gate * and limitations under the License. 13*0Sstevel@tonic-gate * 14*0Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*0Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*0Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*0Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*0Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*0Sstevel@tonic-gate * 20*0Sstevel@tonic-gate * CDDL HEADER END 21*0Sstevel@tonic-gate */ 22*0Sstevel@tonic-gate /* 23*0Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24*0Sstevel@tonic-gate * Use is subject to license terms. 25*0Sstevel@tonic-gate */ 26*0Sstevel@tonic-gate 27*0Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 28*0Sstevel@tonic-gate 29*0Sstevel@tonic-gate #include <sys/types.h> 30*0Sstevel@tonic-gate #include <sys/systm.h> 31*0Sstevel@tonic-gate #include <sys/archsystm.h> 32*0Sstevel@tonic-gate #include <sys/machsystm.h> 33*0Sstevel@tonic-gate #include <sys/t_lock.h> 34*0Sstevel@tonic-gate #include <sys/vmem.h> 35*0Sstevel@tonic-gate #include <sys/mman.h> 36*0Sstevel@tonic-gate #include <sys/vm.h> 37*0Sstevel@tonic-gate #include <sys/cpu.h> 38*0Sstevel@tonic-gate #include <sys/cmn_err.h> 39*0Sstevel@tonic-gate #include <sys/cpuvar.h> 40*0Sstevel@tonic-gate #include <sys/atomic.h> 41*0Sstevel@tonic-gate #include <vm/as.h> 42*0Sstevel@tonic-gate #include <vm/hat.h> 43*0Sstevel@tonic-gate #include <vm/as.h> 44*0Sstevel@tonic-gate #include <vm/page.h> 45*0Sstevel@tonic-gate #include <vm/seg.h> 46*0Sstevel@tonic-gate #include <vm/seg_kmem.h> 47*0Sstevel@tonic-gate #include <vm/hat_sfmmu.h> 48*0Sstevel@tonic-gate #include <sys/debug.h> 49*0Sstevel@tonic-gate #include <sys/cpu_module.h> 50*0Sstevel@tonic-gate 51*0Sstevel@tonic-gate /* 52*0Sstevel@tonic-gate * A quick way to generate a cache consistent address to map in a page. 53*0Sstevel@tonic-gate * users: ppcopy, pagezero, /proc, dev/mem 54*0Sstevel@tonic-gate * 55*0Sstevel@tonic-gate * The ppmapin/ppmapout routines provide a quick way of generating a cache 56*0Sstevel@tonic-gate * consistent address by reserving a given amount of kernel address space. 57*0Sstevel@tonic-gate * The base is PPMAPBASE and its size is PPMAPSIZE. This memory is divided 58*0Sstevel@tonic-gate * into x number of sets, where x is the number of colors for the virtual 59*0Sstevel@tonic-gate * cache. The number of colors is how many times a page can be mapped 60*0Sstevel@tonic-gate * simulatenously in the cache. For direct map caches this translates to 61*0Sstevel@tonic-gate * the number of pages in the cache. 62*0Sstevel@tonic-gate * Each set will be assigned a group of virtual pages from the reserved memory 63*0Sstevel@tonic-gate * depending on its virtual color. 64*0Sstevel@tonic-gate * When trying to assign a virtual address we will find out the color for the 65*0Sstevel@tonic-gate * physical page in question (if applicable). Then we will try to find an 66*0Sstevel@tonic-gate * available virtual page from the set of the appropiate color. 67*0Sstevel@tonic-gate */ 68*0Sstevel@tonic-gate 69*0Sstevel@tonic-gate #define clsettoarray(color, set) ((color * nsets) + set) 70*0Sstevel@tonic-gate 71*0Sstevel@tonic-gate int pp_slots = 4; /* small default, tuned by cpu module */ 72*0Sstevel@tonic-gate 73*0Sstevel@tonic-gate /* tuned by cpu module, default is "safe" */ 74*0Sstevel@tonic-gate int pp_consistent_coloring = PPAGE_STORES_POLLUTE | PPAGE_LOADS_POLLUTE; 75*0Sstevel@tonic-gate 76*0Sstevel@tonic-gate static caddr_t ppmap_vaddrs[PPMAPSIZE / MMU_PAGESIZE]; 77*0Sstevel@tonic-gate static int nsets; /* number of sets */ 78*0Sstevel@tonic-gate static int ppmap_pages; /* generate align mask */ 79*0Sstevel@tonic-gate static int ppmap_shift; /* set selector */ 80*0Sstevel@tonic-gate 81*0Sstevel@tonic-gate #ifdef PPDEBUG 82*0Sstevel@tonic-gate #define MAXCOLORS 16 /* for debug only */ 83*0Sstevel@tonic-gate static int ppalloc_noslot = 0; /* # of allocations from kernelmap */ 84*0Sstevel@tonic-gate static int align_hits[MAXCOLORS]; 85*0Sstevel@tonic-gate static int pp_allocs; /* # of ppmapin requests */ 86*0Sstevel@tonic-gate #endif /* PPDEBUG */ 87*0Sstevel@tonic-gate 88*0Sstevel@tonic-gate /* 89*0Sstevel@tonic-gate * There are only 64 TLB entries on spitfire, 16 on cheetah 90*0Sstevel@tonic-gate * (fully-associative TLB) so we allow the cpu module to tune the 91*0Sstevel@tonic-gate * number to use here via pp_slots. 92*0Sstevel@tonic-gate */ 93*0Sstevel@tonic-gate static struct ppmap_va { 94*0Sstevel@tonic-gate caddr_t ppmap_slots[MAXPP_SLOTS]; 95*0Sstevel@tonic-gate } ppmap_va[NCPU]; 96*0Sstevel@tonic-gate 97*0Sstevel@tonic-gate void 98*0Sstevel@tonic-gate ppmapinit(void) 99*0Sstevel@tonic-gate { 100*0Sstevel@tonic-gate int color, nset, setsize; 101*0Sstevel@tonic-gate caddr_t va; 102*0Sstevel@tonic-gate 103*0Sstevel@tonic-gate ASSERT(pp_slots <= MAXPP_SLOTS); 104*0Sstevel@tonic-gate 105*0Sstevel@tonic-gate va = (caddr_t)PPMAPBASE; 106*0Sstevel@tonic-gate if (cache & CACHE_VAC) { 107*0Sstevel@tonic-gate int a; 108*0Sstevel@tonic-gate 109*0Sstevel@tonic-gate ppmap_pages = mmu_btop(shm_alignment); 110*0Sstevel@tonic-gate nsets = PPMAPSIZE / shm_alignment; 111*0Sstevel@tonic-gate setsize = shm_alignment; 112*0Sstevel@tonic-gate ppmap_shift = MMU_PAGESHIFT; 113*0Sstevel@tonic-gate a = ppmap_pages; 114*0Sstevel@tonic-gate while (a >>= 1) 115*0Sstevel@tonic-gate ppmap_shift++; 116*0Sstevel@tonic-gate } else { 117*0Sstevel@tonic-gate /* 118*0Sstevel@tonic-gate * If we do not have a virtual indexed cache we simply 119*0Sstevel@tonic-gate * have only one set containing all pages. 120*0Sstevel@tonic-gate */ 121*0Sstevel@tonic-gate ppmap_pages = 1; 122*0Sstevel@tonic-gate nsets = mmu_btop(PPMAPSIZE); 123*0Sstevel@tonic-gate setsize = MMU_PAGESIZE; 124*0Sstevel@tonic-gate ppmap_shift = MMU_PAGESHIFT; 125*0Sstevel@tonic-gate } 126*0Sstevel@tonic-gate for (color = 0; color < ppmap_pages; color++) { 127*0Sstevel@tonic-gate for (nset = 0; nset < nsets; nset++) { 128*0Sstevel@tonic-gate ppmap_vaddrs[clsettoarray(color, nset)] = 129*0Sstevel@tonic-gate (caddr_t)((uintptr_t)va + (nset * setsize)); 130*0Sstevel@tonic-gate } 131*0Sstevel@tonic-gate va += MMU_PAGESIZE; 132*0Sstevel@tonic-gate } 133*0Sstevel@tonic-gate } 134*0Sstevel@tonic-gate 135*0Sstevel@tonic-gate /* 136*0Sstevel@tonic-gate * Allocate a cache consistent virtual address to map a page, pp, 137*0Sstevel@tonic-gate * with protection, vprot; and map it in the MMU, using the most 138*0Sstevel@tonic-gate * efficient means possible. The argument avoid is a virtual address 139*0Sstevel@tonic-gate * hint which when masked yields an offset into a virtual cache 140*0Sstevel@tonic-gate * that should be avoided when allocating an address to map in a 141*0Sstevel@tonic-gate * page. An avoid arg of -1 means you don't care, for instance pagezero. 142*0Sstevel@tonic-gate * 143*0Sstevel@tonic-gate * machine dependent, depends on virtual address space layout, 144*0Sstevel@tonic-gate * understands that all kernel addresses have bit 31 set. 145*0Sstevel@tonic-gate * 146*0Sstevel@tonic-gate * NOTE: For sun4 platforms the meaning of the hint argument is opposite from 147*0Sstevel@tonic-gate * that found in other architectures. In other architectures the hint 148*0Sstevel@tonic-gate * (called avoid) was used to ask ppmapin to NOT use the specified cache color. 149*0Sstevel@tonic-gate * This was used to avoid virtual cache trashing in the bcopy. Unfortunately 150*0Sstevel@tonic-gate * in the case of a COW, this later on caused a cache aliasing conflict. In 151*0Sstevel@tonic-gate * sun4, the bcopy routine uses the block ld/st instructions so we don't have 152*0Sstevel@tonic-gate * to worry about virtual cache trashing. Actually, by using the hint to choose 153*0Sstevel@tonic-gate * the right color we can almost guarantee a cache conflict will not occur. 154*0Sstevel@tonic-gate */ 155*0Sstevel@tonic-gate 156*0Sstevel@tonic-gate caddr_t 157*0Sstevel@tonic-gate ppmapin(page_t *pp, uint_t vprot, caddr_t hint) 158*0Sstevel@tonic-gate { 159*0Sstevel@tonic-gate int color, nset, index, start; 160*0Sstevel@tonic-gate caddr_t va; 161*0Sstevel@tonic-gate 162*0Sstevel@tonic-gate #ifdef PPDEBUG 163*0Sstevel@tonic-gate pp_allocs++; 164*0Sstevel@tonic-gate #endif /* PPDEBUG */ 165*0Sstevel@tonic-gate if (cache & CACHE_VAC) { 166*0Sstevel@tonic-gate color = sfmmu_get_ppvcolor(pp); 167*0Sstevel@tonic-gate if (color == -1) { 168*0Sstevel@tonic-gate if ((intptr_t)hint != -1L) { 169*0Sstevel@tonic-gate color = addr_to_vcolor(hint); 170*0Sstevel@tonic-gate } else { 171*0Sstevel@tonic-gate color = addr_to_vcolor(mmu_ptob(pp->p_pagenum)); 172*0Sstevel@tonic-gate } 173*0Sstevel@tonic-gate } 174*0Sstevel@tonic-gate 175*0Sstevel@tonic-gate } else { 176*0Sstevel@tonic-gate /* 177*0Sstevel@tonic-gate * For physical caches, we can pick any address we want. 178*0Sstevel@tonic-gate */ 179*0Sstevel@tonic-gate color = 0; 180*0Sstevel@tonic-gate } 181*0Sstevel@tonic-gate 182*0Sstevel@tonic-gate start = color; 183*0Sstevel@tonic-gate do { 184*0Sstevel@tonic-gate for (nset = 0; nset < nsets; nset++) { 185*0Sstevel@tonic-gate index = clsettoarray(color, nset); 186*0Sstevel@tonic-gate va = ppmap_vaddrs[index]; 187*0Sstevel@tonic-gate if (va != NULL) { 188*0Sstevel@tonic-gate #ifdef PPDEBUG 189*0Sstevel@tonic-gate align_hits[color]++; 190*0Sstevel@tonic-gate #endif /* PPDEBUG */ 191*0Sstevel@tonic-gate if (casptr(&ppmap_vaddrs[index], 192*0Sstevel@tonic-gate va, NULL) == va) { 193*0Sstevel@tonic-gate hat_memload(kas.a_hat, va, pp, 194*0Sstevel@tonic-gate vprot | HAT_NOSYNC, 195*0Sstevel@tonic-gate HAT_LOAD_LOCK); 196*0Sstevel@tonic-gate return (va); 197*0Sstevel@tonic-gate } 198*0Sstevel@tonic-gate } 199*0Sstevel@tonic-gate } 200*0Sstevel@tonic-gate /* 201*0Sstevel@tonic-gate * first pick didn't succeed, try another 202*0Sstevel@tonic-gate */ 203*0Sstevel@tonic-gate if (++color == ppmap_pages) 204*0Sstevel@tonic-gate color = 0; 205*0Sstevel@tonic-gate } while (color != start); 206*0Sstevel@tonic-gate 207*0Sstevel@tonic-gate #ifdef PPDEBUG 208*0Sstevel@tonic-gate ppalloc_noslot++; 209*0Sstevel@tonic-gate #endif /* PPDEBUG */ 210*0Sstevel@tonic-gate 211*0Sstevel@tonic-gate /* 212*0Sstevel@tonic-gate * No free slots; get a random one from the kernel heap area. 213*0Sstevel@tonic-gate */ 214*0Sstevel@tonic-gate va = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP); 215*0Sstevel@tonic-gate 216*0Sstevel@tonic-gate hat_memload(kas.a_hat, va, pp, vprot | HAT_NOSYNC, HAT_LOAD_LOCK); 217*0Sstevel@tonic-gate 218*0Sstevel@tonic-gate return (va); 219*0Sstevel@tonic-gate 220*0Sstevel@tonic-gate } 221*0Sstevel@tonic-gate 222*0Sstevel@tonic-gate void 223*0Sstevel@tonic-gate ppmapout(caddr_t va) 224*0Sstevel@tonic-gate { 225*0Sstevel@tonic-gate int color, nset, index; 226*0Sstevel@tonic-gate 227*0Sstevel@tonic-gate if (va >= kernelheap && va < ekernelheap) { 228*0Sstevel@tonic-gate /* 229*0Sstevel@tonic-gate * Space came from kernelmap, flush the page and 230*0Sstevel@tonic-gate * return the space. 231*0Sstevel@tonic-gate */ 232*0Sstevel@tonic-gate hat_unload(kas.a_hat, va, PAGESIZE, 233*0Sstevel@tonic-gate (HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK)); 234*0Sstevel@tonic-gate vmem_free(heap_arena, va, PAGESIZE); 235*0Sstevel@tonic-gate } else { 236*0Sstevel@tonic-gate /* 237*0Sstevel@tonic-gate * Space came from ppmap_vaddrs[], give it back. 238*0Sstevel@tonic-gate */ 239*0Sstevel@tonic-gate color = addr_to_vcolor(va); 240*0Sstevel@tonic-gate ASSERT((cache & CACHE_VAC)? (color < ppmap_pages) : 1); 241*0Sstevel@tonic-gate 242*0Sstevel@tonic-gate nset = ((uintptr_t)va >> ppmap_shift) & (nsets - 1); 243*0Sstevel@tonic-gate index = clsettoarray(color, nset); 244*0Sstevel@tonic-gate hat_unload(kas.a_hat, va, PAGESIZE, 245*0Sstevel@tonic-gate (HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK)); 246*0Sstevel@tonic-gate 247*0Sstevel@tonic-gate ASSERT(ppmap_vaddrs[index] == NULL); 248*0Sstevel@tonic-gate ppmap_vaddrs[index] = va; 249*0Sstevel@tonic-gate } 250*0Sstevel@tonic-gate } 251*0Sstevel@tonic-gate 252*0Sstevel@tonic-gate #ifdef DEBUG 253*0Sstevel@tonic-gate #define PP_STAT_ADD(stat) (stat)++ 254*0Sstevel@tonic-gate uint_t pload, ploadfail; 255*0Sstevel@tonic-gate uint_t ppzero, ppzero_short; 256*0Sstevel@tonic-gate #else 257*0Sstevel@tonic-gate #define PP_STAT_ADD(stat) 258*0Sstevel@tonic-gate #endif /* DEBUG */ 259*0Sstevel@tonic-gate 260*0Sstevel@tonic-gate /* 261*0Sstevel@tonic-gate * Find a slot in per CPU page copy area. Load up a locked TLB in the 262*0Sstevel@tonic-gate * running cpu. We don't call hat layer to load up the tte since the 263*0Sstevel@tonic-gate * mapping is only temporary. If the thread migrates it'll get a TLB 264*0Sstevel@tonic-gate * miss trap and TLB/TSB miss handler will panic since there is no 265*0Sstevel@tonic-gate * official hat record of this mapping. 266*0Sstevel@tonic-gate */ 267*0Sstevel@tonic-gate static caddr_t 268*0Sstevel@tonic-gate pp_load_tlb(processorid_t cpu, caddr_t **pslot, page_t *pp, uint_t prot) 269*0Sstevel@tonic-gate { 270*0Sstevel@tonic-gate struct ppmap_va *ppmap; 271*0Sstevel@tonic-gate tte_t tte; 272*0Sstevel@tonic-gate caddr_t *myslot; 273*0Sstevel@tonic-gate caddr_t va; 274*0Sstevel@tonic-gate long i, start, stride; 275*0Sstevel@tonic-gate int vcolor; 276*0Sstevel@tonic-gate uint_t flags, strict_flag; 277*0Sstevel@tonic-gate 278*0Sstevel@tonic-gate PP_STAT_ADD(pload); 279*0Sstevel@tonic-gate 280*0Sstevel@tonic-gate ppmap = &ppmap_va[cpu]; 281*0Sstevel@tonic-gate va = (caddr_t)(PPMAP_FAST_BASE + (MMU_PAGESIZE * MAXPP_SLOTS) * cpu); 282*0Sstevel@tonic-gate myslot = ppmap->ppmap_slots; 283*0Sstevel@tonic-gate ASSERT(addr_to_vcolor(va) == 0); 284*0Sstevel@tonic-gate 285*0Sstevel@tonic-gate if (prot & TTE_HWWR_INT) { 286*0Sstevel@tonic-gate flags = PPAGE_STORE_VCOLORING | PPAGE_STORES_POLLUTE; 287*0Sstevel@tonic-gate strict_flag = PPAGE_STORES_POLLUTE; 288*0Sstevel@tonic-gate } else { 289*0Sstevel@tonic-gate flags = PPAGE_LOAD_VCOLORING | PPAGE_LOADS_POLLUTE; 290*0Sstevel@tonic-gate strict_flag = PPAGE_LOADS_POLLUTE; 291*0Sstevel@tonic-gate } 292*0Sstevel@tonic-gate 293*0Sstevel@tonic-gate /* 294*0Sstevel@tonic-gate * If consistent handling is required then keep the current 295*0Sstevel@tonic-gate * vcolor of the page. Furthermore, if loads or stores can 296*0Sstevel@tonic-gate * pollute the VAC then using a "new" page (unassigned vcolor) 297*0Sstevel@tonic-gate * won't work and we have to return a failure. 298*0Sstevel@tonic-gate */ 299*0Sstevel@tonic-gate if (pp_consistent_coloring & flags) { 300*0Sstevel@tonic-gate vcolor = sfmmu_get_ppvcolor(pp); 301*0Sstevel@tonic-gate if ((vcolor == -1) && 302*0Sstevel@tonic-gate (pp_consistent_coloring & strict_flag)) 303*0Sstevel@tonic-gate return (NULL); 304*0Sstevel@tonic-gate /* else keep the current vcolor of the page */ 305*0Sstevel@tonic-gate } else { 306*0Sstevel@tonic-gate vcolor = -1; 307*0Sstevel@tonic-gate } 308*0Sstevel@tonic-gate 309*0Sstevel@tonic-gate if (vcolor != -1) { 310*0Sstevel@tonic-gate va += MMU_PAGESIZE * vcolor; 311*0Sstevel@tonic-gate start = vcolor; 312*0Sstevel@tonic-gate stride = ppmap_pages; /* number of colors */ 313*0Sstevel@tonic-gate myslot += vcolor; 314*0Sstevel@tonic-gate } else { 315*0Sstevel@tonic-gate start = 0; 316*0Sstevel@tonic-gate stride = 1; 317*0Sstevel@tonic-gate } 318*0Sstevel@tonic-gate 319*0Sstevel@tonic-gate for (i = start; i < pp_slots; i += stride) { 320*0Sstevel@tonic-gate if (*myslot == NULL) { 321*0Sstevel@tonic-gate if (casptr(myslot, NULL, va) == NULL) 322*0Sstevel@tonic-gate break; 323*0Sstevel@tonic-gate } 324*0Sstevel@tonic-gate myslot += stride; 325*0Sstevel@tonic-gate va += MMU_PAGESIZE * stride; 326*0Sstevel@tonic-gate } 327*0Sstevel@tonic-gate 328*0Sstevel@tonic-gate if (i >= pp_slots) { 329*0Sstevel@tonic-gate PP_STAT_ADD(ploadfail); 330*0Sstevel@tonic-gate return (NULL); 331*0Sstevel@tonic-gate } 332*0Sstevel@tonic-gate 333*0Sstevel@tonic-gate ASSERT(vcolor == -1 || addr_to_vcolor(va) == vcolor); 334*0Sstevel@tonic-gate 335*0Sstevel@tonic-gate /* 336*0Sstevel@tonic-gate * Now we have a slot we can use, make the tte. 337*0Sstevel@tonic-gate */ 338*0Sstevel@tonic-gate tte.tte_inthi = TTE_VALID_INT | TTE_PFN_INTHI(pp->p_pagenum); 339*0Sstevel@tonic-gate tte.tte_intlo = TTE_PFN_INTLO(pp->p_pagenum) | TTE_CP_INT | 340*0Sstevel@tonic-gate TTE_CV_INT | TTE_PRIV_INT | TTE_LCK_INT | prot; 341*0Sstevel@tonic-gate 342*0Sstevel@tonic-gate ASSERT(CPU->cpu_id == cpu); 343*0Sstevel@tonic-gate sfmmu_dtlb_ld(va, KCONTEXT, &tte); 344*0Sstevel@tonic-gate 345*0Sstevel@tonic-gate *pslot = myslot; /* Return ptr to the slot we used. */ 346*0Sstevel@tonic-gate 347*0Sstevel@tonic-gate return (va); 348*0Sstevel@tonic-gate } 349*0Sstevel@tonic-gate 350*0Sstevel@tonic-gate static void 351*0Sstevel@tonic-gate pp_unload_tlb(caddr_t *pslot, caddr_t va) 352*0Sstevel@tonic-gate { 353*0Sstevel@tonic-gate ASSERT(*pslot == va); 354*0Sstevel@tonic-gate 355*0Sstevel@tonic-gate vtag_flushpage(va, KCONTEXT); 356*0Sstevel@tonic-gate *pslot = NULL; /* release the slot */ 357*0Sstevel@tonic-gate } 358*0Sstevel@tonic-gate 359*0Sstevel@tonic-gate /* 360*0Sstevel@tonic-gate * Common copy routine which attempts to use hwblkpagecopy. If this routine 361*0Sstevel@tonic-gate * can't be used, failure (0) will be returned. Otherwise, a PAGESIZE page 362*0Sstevel@tonic-gate * will be copied and success (1) will be returned. 363*0Sstevel@tonic-gate */ 364*0Sstevel@tonic-gate int 365*0Sstevel@tonic-gate ppcopy_common(page_t *fm_pp, page_t *to_pp) 366*0Sstevel@tonic-gate { 367*0Sstevel@tonic-gate caddr_t fm_va, to_va; 368*0Sstevel@tonic-gate caddr_t *fm_slot, *to_slot; 369*0Sstevel@tonic-gate processorid_t cpu; 370*0Sstevel@tonic-gate 371*0Sstevel@tonic-gate ASSERT(PAGE_LOCKED(fm_pp)); 372*0Sstevel@tonic-gate ASSERT(PAGE_LOCKED(to_pp)); 373*0Sstevel@tonic-gate 374*0Sstevel@tonic-gate /* 375*0Sstevel@tonic-gate * If we can't use VIS block loads and stores we can't use 376*0Sstevel@tonic-gate * pp_load_tlb/pp_unload_tlb due to the possibility of 377*0Sstevel@tonic-gate * d$ aliasing. 378*0Sstevel@tonic-gate */ 379*0Sstevel@tonic-gate if (!use_hw_bcopy && (cache & CACHE_VAC)) 380*0Sstevel@tonic-gate return (0); 381*0Sstevel@tonic-gate 382*0Sstevel@tonic-gate kpreempt_disable(); 383*0Sstevel@tonic-gate cpu = CPU->cpu_id; 384*0Sstevel@tonic-gate fm_va = pp_load_tlb(cpu, &fm_slot, fm_pp, 0); 385*0Sstevel@tonic-gate if (fm_va == NULL) { 386*0Sstevel@tonic-gate kpreempt_enable(); 387*0Sstevel@tonic-gate return (0); 388*0Sstevel@tonic-gate } 389*0Sstevel@tonic-gate to_va = pp_load_tlb(cpu, &to_slot, to_pp, TTE_HWWR_INT); 390*0Sstevel@tonic-gate if (to_va == NULL) { 391*0Sstevel@tonic-gate pp_unload_tlb(fm_slot, fm_va); 392*0Sstevel@tonic-gate kpreempt_enable(); 393*0Sstevel@tonic-gate return (0); 394*0Sstevel@tonic-gate } 395*0Sstevel@tonic-gate hwblkpagecopy(fm_va, to_va); 396*0Sstevel@tonic-gate ASSERT(CPU->cpu_id == cpu); 397*0Sstevel@tonic-gate pp_unload_tlb(fm_slot, fm_va); 398*0Sstevel@tonic-gate pp_unload_tlb(to_slot, to_va); 399*0Sstevel@tonic-gate kpreempt_enable(); 400*0Sstevel@tonic-gate return (1); 401*0Sstevel@tonic-gate } 402*0Sstevel@tonic-gate 403*0Sstevel@tonic-gate /* 404*0Sstevel@tonic-gate * Routine to copy kernel pages during relocation. It will copy one 405*0Sstevel@tonic-gate * PAGESIZE page to another PAGESIZE page. This function may be called 406*0Sstevel@tonic-gate * above LOCK_LEVEL so it should not grab any locks. 407*0Sstevel@tonic-gate */ 408*0Sstevel@tonic-gate void 409*0Sstevel@tonic-gate ppcopy_kernel__relocatable(page_t *fm_pp, page_t *to_pp) 410*0Sstevel@tonic-gate { 411*0Sstevel@tonic-gate uint64_t fm_pa, to_pa; 412*0Sstevel@tonic-gate size_t nbytes; 413*0Sstevel@tonic-gate 414*0Sstevel@tonic-gate fm_pa = (uint64_t)(fm_pp->p_pagenum) << MMU_PAGESHIFT; 415*0Sstevel@tonic-gate to_pa = (uint64_t)(to_pp->p_pagenum) << MMU_PAGESHIFT; 416*0Sstevel@tonic-gate 417*0Sstevel@tonic-gate nbytes = MMU_PAGESIZE; 418*0Sstevel@tonic-gate 419*0Sstevel@tonic-gate for (; nbytes > 0; fm_pa += 32, to_pa += 32, nbytes -= 32) 420*0Sstevel@tonic-gate hw_pa_bcopy32(fm_pa, to_pa); 421*0Sstevel@tonic-gate } 422*0Sstevel@tonic-gate 423*0Sstevel@tonic-gate /* 424*0Sstevel@tonic-gate * Copy the data from the physical page represented by "frompp" to 425*0Sstevel@tonic-gate * that represented by "topp". 426*0Sstevel@tonic-gate * 427*0Sstevel@tonic-gate * Try to use per cpu mapping first, if that fails then call pp_mapin 428*0Sstevel@tonic-gate * to load it. 429*0Sstevel@tonic-gate */ 430*0Sstevel@tonic-gate void 431*0Sstevel@tonic-gate ppcopy(page_t *fm_pp, page_t *to_pp) 432*0Sstevel@tonic-gate { 433*0Sstevel@tonic-gate caddr_t fm_va, to_va; 434*0Sstevel@tonic-gate 435*0Sstevel@tonic-gate /* Try the fast path first */ 436*0Sstevel@tonic-gate if (ppcopy_common(fm_pp, to_pp)) 437*0Sstevel@tonic-gate return; 438*0Sstevel@tonic-gate 439*0Sstevel@tonic-gate /* Fast path failed, so we need to do the slow path. */ 440*0Sstevel@tonic-gate fm_va = ppmapin(fm_pp, PROT_READ, (caddr_t)-1); 441*0Sstevel@tonic-gate to_va = ppmapin(to_pp, PROT_READ | PROT_WRITE, fm_va); 442*0Sstevel@tonic-gate bcopy(fm_va, to_va, PAGESIZE); 443*0Sstevel@tonic-gate ppmapout(fm_va); 444*0Sstevel@tonic-gate ppmapout(to_va); 445*0Sstevel@tonic-gate } 446*0Sstevel@tonic-gate 447*0Sstevel@tonic-gate /* 448*0Sstevel@tonic-gate * Zero the physical page from off to off + len given by `pp' 449*0Sstevel@tonic-gate * without changing the reference and modified bits of page. 450*0Sstevel@tonic-gate * 451*0Sstevel@tonic-gate * Again, we'll try per cpu mapping first. 452*0Sstevel@tonic-gate */ 453*0Sstevel@tonic-gate void 454*0Sstevel@tonic-gate pagezero(page_t *pp, uint_t off, uint_t len) 455*0Sstevel@tonic-gate { 456*0Sstevel@tonic-gate caddr_t va; 457*0Sstevel@tonic-gate caddr_t *slot; 458*0Sstevel@tonic-gate int fast = 1; 459*0Sstevel@tonic-gate processorid_t cpu; 460*0Sstevel@tonic-gate extern int hwblkclr(void *, size_t); 461*0Sstevel@tonic-gate extern int use_hw_bzero; 462*0Sstevel@tonic-gate 463*0Sstevel@tonic-gate ASSERT((int)len > 0 && (int)off >= 0 && off + len <= PAGESIZE); 464*0Sstevel@tonic-gate ASSERT(PAGE_LOCKED(pp)); 465*0Sstevel@tonic-gate 466*0Sstevel@tonic-gate PP_STAT_ADD(ppzero); 467*0Sstevel@tonic-gate 468*0Sstevel@tonic-gate if (len != MMU_PAGESIZE || !use_hw_bzero) { 469*0Sstevel@tonic-gate /* 470*0Sstevel@tonic-gate * Since the fast path doesn't do anything about 471*0Sstevel@tonic-gate * VAC coloring, we make sure bcopy h/w will be used. 472*0Sstevel@tonic-gate */ 473*0Sstevel@tonic-gate fast = 0; 474*0Sstevel@tonic-gate va = NULL; 475*0Sstevel@tonic-gate PP_STAT_ADD(ppzero_short); 476*0Sstevel@tonic-gate } 477*0Sstevel@tonic-gate 478*0Sstevel@tonic-gate kpreempt_disable(); 479*0Sstevel@tonic-gate 480*0Sstevel@tonic-gate if (fast) { 481*0Sstevel@tonic-gate cpu = CPU->cpu_id; 482*0Sstevel@tonic-gate va = pp_load_tlb(cpu, &slot, pp, TTE_HWWR_INT); 483*0Sstevel@tonic-gate } 484*0Sstevel@tonic-gate 485*0Sstevel@tonic-gate if (va == NULL) { 486*0Sstevel@tonic-gate /* 487*0Sstevel@tonic-gate * We are here either length != MMU_PAGESIZE or pp_load_tlb() 488*0Sstevel@tonic-gate * returns NULL or use_hw_bzero is disabled. 489*0Sstevel@tonic-gate */ 490*0Sstevel@tonic-gate va = ppmapin(pp, PROT_READ | PROT_WRITE, (caddr_t)-1); 491*0Sstevel@tonic-gate fast = 0; 492*0Sstevel@tonic-gate } 493*0Sstevel@tonic-gate 494*0Sstevel@tonic-gate if (hwblkclr(va + off, len)) { 495*0Sstevel@tonic-gate /* 496*0Sstevel@tonic-gate * We may not have used block commit asi. 497*0Sstevel@tonic-gate * So flush the I-$ manually 498*0Sstevel@tonic-gate */ 499*0Sstevel@tonic-gate 500*0Sstevel@tonic-gate ASSERT(fast == 0); 501*0Sstevel@tonic-gate 502*0Sstevel@tonic-gate sync_icache(va + off, len); 503*0Sstevel@tonic-gate } else { 504*0Sstevel@tonic-gate /* 505*0Sstevel@tonic-gate * We have used blk commit, and flushed the I-$. However we 506*0Sstevel@tonic-gate * still may have an instruction in the pipeline. Only a flush 507*0Sstevel@tonic-gate * instruction will invalidate that. 508*0Sstevel@tonic-gate */ 509*0Sstevel@tonic-gate doflush(va); 510*0Sstevel@tonic-gate } 511*0Sstevel@tonic-gate 512*0Sstevel@tonic-gate if (fast) { 513*0Sstevel@tonic-gate ASSERT(CPU->cpu_id == cpu); 514*0Sstevel@tonic-gate pp_unload_tlb(slot, va); 515*0Sstevel@tonic-gate } else { 516*0Sstevel@tonic-gate ppmapout(va); 517*0Sstevel@tonic-gate } 518*0Sstevel@tonic-gate 519*0Sstevel@tonic-gate kpreempt_enable(); 520*0Sstevel@tonic-gate } 521