10Sstevel@tonic-gate /* 20Sstevel@tonic-gate * CDDL HEADER START 30Sstevel@tonic-gate * 40Sstevel@tonic-gate * The contents of this file are subject to the terms of the 50Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 60Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 70Sstevel@tonic-gate * with the License. 80Sstevel@tonic-gate * 90Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 100Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 110Sstevel@tonic-gate * See the License for the specific language governing permissions 120Sstevel@tonic-gate * and limitations under the License. 130Sstevel@tonic-gate * 140Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 150Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 160Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 170Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 180Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 190Sstevel@tonic-gate * 200Sstevel@tonic-gate * CDDL HEADER END 210Sstevel@tonic-gate */ 220Sstevel@tonic-gate /* 231373Skchow * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 240Sstevel@tonic-gate * Use is subject to license terms. 250Sstevel@tonic-gate */ 260Sstevel@tonic-gate 270Sstevel@tonic-gate /* 280Sstevel@tonic-gate * UNIX machine dependent virtual memory support. 290Sstevel@tonic-gate */ 300Sstevel@tonic-gate 310Sstevel@tonic-gate #ifndef _VM_DEP_H 320Sstevel@tonic-gate #define _VM_DEP_H 330Sstevel@tonic-gate 340Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 350Sstevel@tonic-gate 360Sstevel@tonic-gate #ifdef __cplusplus 370Sstevel@tonic-gate extern "C" { 380Sstevel@tonic-gate #endif 390Sstevel@tonic-gate 400Sstevel@tonic-gate #include <vm/hat_sfmmu.h> 410Sstevel@tonic-gate #include <sys/archsystm.h> 420Sstevel@tonic-gate #include <sys/memnode.h> 430Sstevel@tonic-gate 440Sstevel@tonic-gate #define GETTICK() gettick() 450Sstevel@tonic-gate 460Sstevel@tonic-gate /* 470Sstevel@tonic-gate * Per page size free lists. Allocated dynamically. 480Sstevel@tonic-gate */ 490Sstevel@tonic-gate #define MAX_MEM_TYPES 2 /* 0 = reloc, 1 = noreloc */ 500Sstevel@tonic-gate #define MTYPE_RELOC 0 510Sstevel@tonic-gate #define MTYPE_NORELOC 1 520Sstevel@tonic-gate 530Sstevel@tonic-gate #define PP_2_MTYPE(pp) (PP_ISNORELOC(pp) ? MTYPE_NORELOC : MTYPE_RELOC) 540Sstevel@tonic-gate 55*1385Skchow #define MTYPE_INIT(mtype, vp, vaddr, flags, pgsz) \ 560Sstevel@tonic-gate mtype = (flags & PG_NORELOC) ? MTYPE_NORELOC : MTYPE_RELOC; 570Sstevel@tonic-gate 580Sstevel@tonic-gate /* mtype init for page_get_replacement_page */ 590Sstevel@tonic-gate 60*1385Skchow #define MTYPE_PGR_INIT(mtype, flags, pp, mnode, pgcnt) \ 610Sstevel@tonic-gate mtype = (flags & PG_NORELOC) ? MTYPE_NORELOC : MTYPE_RELOC; 620Sstevel@tonic-gate 630Sstevel@tonic-gate #define MNODETYPE_2_PFN(mnode, mtype, pfnlo, pfnhi) \ 640Sstevel@tonic-gate ASSERT(mtype != MTYPE_NORELOC); \ 650Sstevel@tonic-gate pfnlo = mem_node_config[mnode].physbase; \ 660Sstevel@tonic-gate pfnhi = mem_node_config[mnode].physmax; 670Sstevel@tonic-gate 680Sstevel@tonic-gate /* 690Sstevel@tonic-gate * Internal PG_ flags. 700Sstevel@tonic-gate */ 710Sstevel@tonic-gate #define PGI_RELOCONLY 0x10000 /* acts in the opposite sense to PG_NORELOC */ 720Sstevel@tonic-gate #define PGI_NOCAGE 0x20000 /* indicates Cage is disabled */ 730Sstevel@tonic-gate #define PGI_PGCPHIPRI 0x40000 /* page_get_contig_page priority allocation */ 740Sstevel@tonic-gate #define PGI_PGCPSZC0 0x80000 /* relocate base pagesize page */ 750Sstevel@tonic-gate 760Sstevel@tonic-gate /* 770Sstevel@tonic-gate * PGI mtype flags - should not overlap PGI flags 780Sstevel@tonic-gate */ 790Sstevel@tonic-gate #define PGI_MT_RANGE 0x1000000 /* mtype range */ 800Sstevel@tonic-gate #define PGI_MT_NEXT 0x2000000 /* get next mtype */ 810Sstevel@tonic-gate 820Sstevel@tonic-gate extern page_t ***page_freelists[MMU_PAGE_SIZES][MAX_MEM_TYPES]; 830Sstevel@tonic-gate extern page_t ***page_cachelists[MAX_MEM_TYPES]; 840Sstevel@tonic-gate 850Sstevel@tonic-gate #define PAGE_FREELISTS(mnode, szc, color, mtype) \ 860Sstevel@tonic-gate (*(page_freelists[szc][mtype][mnode] + (color))) 870Sstevel@tonic-gate 880Sstevel@tonic-gate #define PAGE_CACHELISTS(mnode, color, mtype) \ 890Sstevel@tonic-gate (*(page_cachelists[mtype][mnode] + (color))) 900Sstevel@tonic-gate 910Sstevel@tonic-gate /* 920Sstevel@tonic-gate * There are 'page_colors' colors/bins. Spread them out under a 930Sstevel@tonic-gate * couple of locks. There are mutexes for both the page freelist 940Sstevel@tonic-gate * and the page cachelist. We want enough locks to make contention 950Sstevel@tonic-gate * reasonable, but not too many -- otherwise page_freelist_lock() gets 960Sstevel@tonic-gate * so expensive that it becomes the bottleneck! 970Sstevel@tonic-gate */ 980Sstevel@tonic-gate #define NPC_MUTEX 16 990Sstevel@tonic-gate 1000Sstevel@tonic-gate extern kmutex_t *fpc_mutex[NPC_MUTEX]; 1010Sstevel@tonic-gate extern kmutex_t *cpc_mutex[NPC_MUTEX]; 1020Sstevel@tonic-gate 1030Sstevel@tonic-gate /* Find the bin for the given page if it was of size szc */ 1040Sstevel@tonic-gate #define PP_2_BIN_SZC(pp, szc) \ 1050Sstevel@tonic-gate (((pp->p_pagenum) & page_colors_mask) >> \ 1060Sstevel@tonic-gate (hw_page_array[szc].hp_shift - hw_page_array[0].hp_shift)) 1070Sstevel@tonic-gate 1080Sstevel@tonic-gate #define PP_2_BIN(pp) (PP_2_BIN_SZC(pp, pp->p_szc)) 1090Sstevel@tonic-gate 1100Sstevel@tonic-gate #define PP_2_MEM_NODE(pp) (PFN_2_MEM_NODE(pp->p_pagenum)) 1110Sstevel@tonic-gate 1120Sstevel@tonic-gate #define PC_BIN_MUTEX(mnode, bin, flags) ((flags & PG_FREE_LIST) ? \ 1130Sstevel@tonic-gate &fpc_mutex[(bin) & (NPC_MUTEX - 1)][mnode] : \ 1140Sstevel@tonic-gate &cpc_mutex[(bin) & (NPC_MUTEX - 1)][mnode]) 1150Sstevel@tonic-gate 1160Sstevel@tonic-gate #define FPC_MUTEX(mnode, i) (&fpc_mutex[i][mnode]) 1170Sstevel@tonic-gate #define CPC_MUTEX(mnode, i) (&cpc_mutex[i][mnode]) 1180Sstevel@tonic-gate 1190Sstevel@tonic-gate #define PFN_BASE(pfnum, szc) (pfnum & ~((1 << PAGE_BSZS_SHIFT(szc)) - 1)) 1200Sstevel@tonic-gate 1210Sstevel@tonic-gate typedef char hpmctr_t; 1220Sstevel@tonic-gate 1230Sstevel@tonic-gate #ifdef DEBUG 1240Sstevel@tonic-gate #define CHK_LPG(pp, szc) chk_lpg(pp, szc) 1250Sstevel@tonic-gate extern void chk_lpg(page_t *, uchar_t); 1260Sstevel@tonic-gate #else 1270Sstevel@tonic-gate #define CHK_LPG(pp, szc) 1280Sstevel@tonic-gate #endif 1290Sstevel@tonic-gate 130414Skchow /* 131414Skchow * page list count per mnode and type. 132414Skchow */ 1330Sstevel@tonic-gate typedef struct { 134414Skchow pgcnt_t plc_mt_pgmax; /* max page cnt */ 135414Skchow pgcnt_t plc_mt_clpgcnt; /* cache list cnt */ 136414Skchow pgcnt_t plc_mt_flpgcnt; /* free list cnt - small pages */ 137414Skchow pgcnt_t plc_mt_lgpgcnt; /* free list cnt - large pages */ 138414Skchow #ifdef DEBUG 1390Sstevel@tonic-gate struct { 140414Skchow pgcnt_t plc_mts_pgcnt; /* per page size count */ 141414Skchow int plc_mts_colors; 142414Skchow pgcnt_t *plc_mtsc_pgcnt; /* per color bin count */ 143414Skchow } plc_mts[MMU_PAGE_SIZES]; 144414Skchow #endif 145414Skchow } plcnt_t[MAX_MEM_NODES][MAX_MEM_TYPES]; 1460Sstevel@tonic-gate 147414Skchow #ifdef DEBUG 1480Sstevel@tonic-gate 1490Sstevel@tonic-gate #define PLCNT_SZ(ctrs_sz) { \ 1500Sstevel@tonic-gate int szc; \ 1510Sstevel@tonic-gate for (szc = 0; szc <= mmu_page_sizes; szc++) { \ 1520Sstevel@tonic-gate int colors = page_get_pagecolors(szc); \ 1530Sstevel@tonic-gate ctrs_sz += (max_mem_nodes * MAX_MEM_TYPES * \ 1540Sstevel@tonic-gate colors * sizeof (pgcnt_t)); \ 1550Sstevel@tonic-gate } \ 1560Sstevel@tonic-gate } 1570Sstevel@tonic-gate 1580Sstevel@tonic-gate #define PLCNT_INIT(base) { \ 1590Sstevel@tonic-gate int mn, mt, szc, colors; \ 1600Sstevel@tonic-gate for (szc = 0; szc < mmu_page_sizes; szc++) { \ 1610Sstevel@tonic-gate colors = page_get_pagecolors(szc); \ 1620Sstevel@tonic-gate for (mn = 0; mn < max_mem_nodes; mn++) { \ 1630Sstevel@tonic-gate for (mt = 0; mt < MAX_MEM_TYPES; mt++) { \ 164*1385Skchow plcnt[mn][mt].plc_mts[szc]. \ 1650Sstevel@tonic-gate plc_mts_colors = colors; \ 166*1385Skchow plcnt[mn][mt].plc_mts[szc]. \ 1670Sstevel@tonic-gate plc_mtsc_pgcnt = (pgcnt_t *)base; \ 1680Sstevel@tonic-gate base += (colors * sizeof (pgcnt_t)); \ 1690Sstevel@tonic-gate } \ 1700Sstevel@tonic-gate } \ 1710Sstevel@tonic-gate } \ 1720Sstevel@tonic-gate } 1730Sstevel@tonic-gate 174414Skchow #define PLCNT_DO(pp, mn, mtype, szc, cnt, flags) { \ 1750Sstevel@tonic-gate int bin = PP_2_BIN(pp); \ 1760Sstevel@tonic-gate if (flags & PG_CACHE_LIST) \ 177414Skchow atomic_add_long(&plcnt[mn][mtype].plc_mt_clpgcnt, cnt); \ 178414Skchow else if (szc) \ 179414Skchow atomic_add_long(&plcnt[mn][mtype].plc_mt_lgpgcnt, cnt); \ 180414Skchow else \ 181414Skchow atomic_add_long(&plcnt[mn][mtype].plc_mt_flpgcnt, cnt); \ 182414Skchow atomic_add_long(&plcnt[mn][mtype].plc_mts[szc].plc_mts_pgcnt, \ 183414Skchow cnt); \ 184414Skchow atomic_add_long(&plcnt[mn][mtype].plc_mts[szc]. \ 1850Sstevel@tonic-gate plc_mtsc_pgcnt[bin], cnt); \ 1860Sstevel@tonic-gate } 1870Sstevel@tonic-gate 1880Sstevel@tonic-gate #else 1890Sstevel@tonic-gate 1900Sstevel@tonic-gate #define PLCNT_SZ(ctrs_sz) 1910Sstevel@tonic-gate 1920Sstevel@tonic-gate #define PLCNT_INIT(base) 1930Sstevel@tonic-gate 194414Skchow /* PG_FREE_LIST may not be explicitly set in flags for large pages */ 1950Sstevel@tonic-gate 196414Skchow #define PLCNT_DO(pp, mn, mtype, szc, cnt, flags) { \ 197414Skchow if (flags & PG_CACHE_LIST) \ 198414Skchow atomic_add_long(&plcnt[mn][mtype].plc_mt_clpgcnt, cnt); \ 199414Skchow else if (szc) \ 200414Skchow atomic_add_long(&plcnt[mn][mtype].plc_mt_lgpgcnt, cnt); \ 201414Skchow else \ 202414Skchow atomic_add_long(&plcnt[mn][mtype].plc_mt_flpgcnt, cnt); \ 2030Sstevel@tonic-gate } 2040Sstevel@tonic-gate 2050Sstevel@tonic-gate #endif 2060Sstevel@tonic-gate 207414Skchow #define PLCNT_INCR(pp, mn, mtype, szc, flags) { \ 208414Skchow long cnt = (1 << PAGE_BSZS_SHIFT(szc)); \ 209414Skchow PLCNT_DO(pp, mn, mtype, szc, cnt, flags); \ 210414Skchow } 211414Skchow 212414Skchow #define PLCNT_DECR(pp, mn, mtype, szc, flags) { \ 213414Skchow long cnt = ((-1) << PAGE_BSZS_SHIFT(szc)); \ 214414Skchow PLCNT_DO(pp, mn, mtype, szc, cnt, flags); \ 215414Skchow } 216414Skchow 217414Skchow /* 218414Skchow * macros to update page list max counts - done when pages transferred 2191373Skchow * from RELOC to NORELOC mtype (kcage_init or kcage_assimilate_page). 220414Skchow */ 2211373Skchow 2221373Skchow #define PLCNT_XFER_NORELOC(pp) { \ 2231373Skchow long cnt = (1 << PAGE_BSZS_SHIFT((pp)->p_szc)); \ 2241373Skchow int mn = PP_2_MEM_NODE(pp); \ 2251373Skchow atomic_add_long(&plcnt[mn][MTYPE_NORELOC].plc_mt_pgmax, cnt); \ 2261373Skchow atomic_add_long(&plcnt[mn][MTYPE_RELOC].plc_mt_pgmax, -cnt); \ 227414Skchow } 228414Skchow 2291373Skchow /* 2301373Skchow * macro to modify the page list max counts when memory is added to 2311373Skchow * the page lists during startup (add_physmem) or during a DR operation 2321373Skchow * when memory is added (kphysm_add_memory_dynamic) or deleted 2331373Skchow * (kphysm_del_cleanup). 2341373Skchow */ 2351373Skchow #define PLCNT_MODIFY_MAX(pfn, cnt) { \ 2361373Skchow int mn = PFN_2_MEM_NODE(pfn); \ 2371373Skchow atomic_add_long(&plcnt[mn][MTYPE_RELOC].plc_mt_pgmax, (cnt)); \ 238414Skchow } 239414Skchow 240414Skchow extern plcnt_t plcnt; 241414Skchow 242414Skchow #define MNODE_PGCNT(mn) \ 243414Skchow (plcnt[mn][MTYPE_RELOC].plc_mt_clpgcnt + \ 244414Skchow plcnt[mn][MTYPE_NORELOC].plc_mt_clpgcnt + \ 245414Skchow plcnt[mn][MTYPE_RELOC].plc_mt_flpgcnt + \ 246414Skchow plcnt[mn][MTYPE_NORELOC].plc_mt_flpgcnt + \ 247414Skchow plcnt[mn][MTYPE_RELOC].plc_mt_lgpgcnt + \ 248414Skchow plcnt[mn][MTYPE_NORELOC].plc_mt_lgpgcnt) 249414Skchow 250414Skchow #define MNODETYPE_PGCNT(mn, mtype) \ 251414Skchow (plcnt[mn][mtype].plc_mt_clpgcnt + \ 252414Skchow plcnt[mn][mtype].plc_mt_flpgcnt + \ 253414Skchow plcnt[mn][mtype].plc_mt_lgpgcnt) 254414Skchow 255414Skchow /* 256414Skchow * macros to loop through the mtype range - MTYPE_START returns -1 in 257414Skchow * mtype if no pages in mnode/mtype and possibly NEXT mtype. 258414Skchow */ 259414Skchow #define MTYPE_START(mnode, mtype, flags) { \ 260414Skchow if (plcnt[mnode][mtype].plc_mt_pgmax == 0) { \ 261414Skchow ASSERT(MNODETYPE_PGCNT(mnode, mtype) == 0); \ 262414Skchow MTYPE_NEXT(mnode, mtype, flags); \ 263414Skchow } \ 264414Skchow } 265414Skchow 266414Skchow /* 267414Skchow * if allocation from the RELOC pool failed and there is sufficient cage 268414Skchow * memory, attempt to allocate from the NORELOC pool. 269414Skchow */ 270414Skchow #define MTYPE_NEXT(mnode, mtype, flags) { \ 271414Skchow if (!(flags & (PG_NORELOC | PGI_NOCAGE | PGI_RELOCONLY)) && \ 272414Skchow (kcage_freemem >= kcage_lotsfree)) { \ 273414Skchow if (plcnt[mnode][mtype].plc_mt_pgmax == 0) { \ 274414Skchow ASSERT(MNODETYPE_PGCNT(mnode, mtype) == 0); \ 275414Skchow mtype = -1; \ 276414Skchow } else { \ 277414Skchow mtype = MTYPE_NORELOC; \ 278414Skchow flags |= PG_NORELOC; \ 279414Skchow } \ 280414Skchow } else { \ 281414Skchow mtype = -1; \ 282414Skchow } \ 283414Skchow } 284414Skchow 2850Sstevel@tonic-gate /* 2860Sstevel@tonic-gate * get the ecache setsize for the current cpu. 2870Sstevel@tonic-gate */ 2880Sstevel@tonic-gate #define CPUSETSIZE() (cpunodes[CPU->cpu_id].ecache_setsize) 2890Sstevel@tonic-gate 290414Skchow extern struct cpu cpu0; 291414Skchow #define CPU0 &cpu0 2920Sstevel@tonic-gate 2930Sstevel@tonic-gate #define PAGE_BSZS_SHIFT(szc) TTE_BSZS_SHIFT(szc) 2940Sstevel@tonic-gate /* 2950Sstevel@tonic-gate * For sfmmu each larger page is 8 times the size of the previous 2960Sstevel@tonic-gate * size page. 2970Sstevel@tonic-gate */ 2980Sstevel@tonic-gate #define FULL_REGION_CNT(rg_szc) (8) 2990Sstevel@tonic-gate 3000Sstevel@tonic-gate /* 3010Sstevel@tonic-gate * The counter base must be per page_counter element to prevent 3020Sstevel@tonic-gate * races when re-indexing, and the base page size element should 3030Sstevel@tonic-gate * be aligned on a boundary of the given region size. 3040Sstevel@tonic-gate * 3050Sstevel@tonic-gate * We also round up the number of pages spanned by the counters 3060Sstevel@tonic-gate * for a given region to PC_BASE_ALIGN in certain situations to simplify 3070Sstevel@tonic-gate * the coding for some non-performance critical routines. 3080Sstevel@tonic-gate */ 3090Sstevel@tonic-gate #define PC_BASE_ALIGN ((pfn_t)1 << PAGE_BSZS_SHIFT(mmu_page_sizes-1)) 3100Sstevel@tonic-gate #define PC_BASE_ALIGN_MASK (PC_BASE_ALIGN - 1) 3110Sstevel@tonic-gate 3120Sstevel@tonic-gate extern int ecache_alignsize; 3130Sstevel@tonic-gate #define L2CACHE_ALIGN ecache_alignsize 314450Skchow #define L2CACHE_ALIGN_MAX 512 3150Sstevel@tonic-gate 3160Sstevel@tonic-gate extern int consistent_coloring; 3170Sstevel@tonic-gate extern uint_t vac_colors_mask; 3180Sstevel@tonic-gate extern int vac_size; 3190Sstevel@tonic-gate extern int vac_shift; 3200Sstevel@tonic-gate 3210Sstevel@tonic-gate /* 3220Sstevel@tonic-gate * Auto large page selection support variables. Some CPU 3230Sstevel@tonic-gate * implementations may differ from the defaults and will need 3240Sstevel@tonic-gate * to change these. 3250Sstevel@tonic-gate */ 3260Sstevel@tonic-gate extern int auto_lpg_tlb_threshold; 3270Sstevel@tonic-gate extern int auto_lpg_minszc; 3280Sstevel@tonic-gate extern int auto_lpg_maxszc; 3290Sstevel@tonic-gate extern size_t auto_lpg_heap_default; 3300Sstevel@tonic-gate extern size_t auto_lpg_stack_default; 3310Sstevel@tonic-gate extern size_t auto_lpg_va_default; 3320Sstevel@tonic-gate extern size_t auto_lpg_remap_threshold; 333423Sdavemq extern pgcnt_t auto_lpg_min_physmem; 3340Sstevel@tonic-gate 3350Sstevel@tonic-gate /* 3360Sstevel@tonic-gate * AS_2_BIN macro controls the page coloring policy. 3370Sstevel@tonic-gate * 0 (default) uses various vaddr bits 3380Sstevel@tonic-gate * 1 virtual=paddr 3390Sstevel@tonic-gate * 2 bin hopping 3400Sstevel@tonic-gate */ 3410Sstevel@tonic-gate #define AS_2_BIN(as, seg, vp, addr, bin) \ 3420Sstevel@tonic-gate switch (consistent_coloring) { \ 3430Sstevel@tonic-gate default: \ 3440Sstevel@tonic-gate cmn_err(CE_WARN, \ 3450Sstevel@tonic-gate "AS_2_BIN: bad consistent coloring value"); \ 3460Sstevel@tonic-gate /* assume default algorithm -> continue */ \ 3470Sstevel@tonic-gate case 0: { \ 3480Sstevel@tonic-gate uint32_t ndx, new; \ 3490Sstevel@tonic-gate int slew = 0; \ 3500Sstevel@tonic-gate \ 3510Sstevel@tonic-gate if (vp != NULL && IS_SWAPVP(vp) && \ 3520Sstevel@tonic-gate seg->s_ops == &segvn_ops) \ 3530Sstevel@tonic-gate slew = as_color_bin(as); \ 3540Sstevel@tonic-gate \ 3550Sstevel@tonic-gate bin = (((uintptr_t)addr >> MMU_PAGESHIFT) + \ 3560Sstevel@tonic-gate (((uintptr_t)addr >> page_coloring_shift) << \ 3570Sstevel@tonic-gate (vac_shift - MMU_PAGESHIFT)) + slew) & \ 3580Sstevel@tonic-gate page_colors_mask; \ 3590Sstevel@tonic-gate \ 3600Sstevel@tonic-gate break; \ 3610Sstevel@tonic-gate } \ 3620Sstevel@tonic-gate case 1: \ 3630Sstevel@tonic-gate bin = ((uintptr_t)addr >> MMU_PAGESHIFT) & \ 3640Sstevel@tonic-gate page_colors_mask; \ 3650Sstevel@tonic-gate break; \ 3660Sstevel@tonic-gate case 2: { \ 3670Sstevel@tonic-gate int cnt = as_color_bin(as); \ 3680Sstevel@tonic-gate /* make sure physical color aligns with vac color */ \ 3690Sstevel@tonic-gate while ((cnt & vac_colors_mask) != \ 3700Sstevel@tonic-gate addr_to_vcolor(addr)) { \ 3710Sstevel@tonic-gate cnt++; \ 3720Sstevel@tonic-gate } \ 3730Sstevel@tonic-gate bin = cnt = cnt & page_colors_mask; \ 3740Sstevel@tonic-gate /* update per as page coloring fields */ \ 3750Sstevel@tonic-gate cnt = (cnt + 1) & page_colors_mask; \ 3760Sstevel@tonic-gate if (cnt == (as_color_start(as) & page_colors_mask)) { \ 3770Sstevel@tonic-gate cnt = as_color_start(as) = as_color_start(as) + \ 3780Sstevel@tonic-gate PGCLR_LOOPFACTOR; \ 3790Sstevel@tonic-gate } \ 3800Sstevel@tonic-gate as_color_bin(as) = cnt & page_colors_mask; \ 3810Sstevel@tonic-gate break; \ 3820Sstevel@tonic-gate } \ 3830Sstevel@tonic-gate } \ 3840Sstevel@tonic-gate ASSERT(bin <= page_colors_mask); 3850Sstevel@tonic-gate 3860Sstevel@tonic-gate /* 387414Skchow * cpu private vm data - accessed thru CPU->cpu_vm_data 388414Skchow * vc_pnum_memseg: tracks last memseg visited in page_numtopp_nolock() 389414Skchow * vc_pnext_memseg: tracks last memseg visited in page_nextn() 390414Skchow * vc_kmptr: unaligned kmem pointer for this vm_cpu_data_t 391450Skchow * vc_kmsize: orignal kmem size for this vm_cpu_data_t 392414Skchow */ 393414Skchow 394414Skchow typedef struct { 395414Skchow struct memseg *vc_pnum_memseg; 396414Skchow struct memseg *vc_pnext_memseg; 397414Skchow void *vc_kmptr; 398450Skchow size_t vc_kmsize; 399414Skchow } vm_cpu_data_t; 400414Skchow 401414Skchow /* allocation size to ensure vm_cpu_data_t resides in its own cache line */ 402414Skchow #define VM_CPU_DATA_PADSIZE \ 403414Skchow (P2ROUNDUP(sizeof (vm_cpu_data_t), L2CACHE_ALIGN_MAX)) 404414Skchow 405414Skchow /* for boot cpu before kmem is initialized */ 406414Skchow extern char vm_cpu_data0[]; 407414Skchow 408414Skchow /* 4090Sstevel@tonic-gate * Function to get an ecache color bin: F(as, cnt, vcolor). 4100Sstevel@tonic-gate * the goal of this function is to: 4110Sstevel@tonic-gate * - to spread a processes' physical pages across the entire ecache to 4120Sstevel@tonic-gate * maximize its use. 4130Sstevel@tonic-gate * - to minimize vac flushes caused when we reuse a physical page on a 4140Sstevel@tonic-gate * different vac color than it was previously used. 4150Sstevel@tonic-gate * - to prevent all processes to use the same exact colors and trash each 4160Sstevel@tonic-gate * other. 4170Sstevel@tonic-gate * 4180Sstevel@tonic-gate * cnt is a bin ptr kept on a per as basis. As we page_create we increment 4190Sstevel@tonic-gate * the ptr so we spread out the physical pages to cover the entire ecache. 4200Sstevel@tonic-gate * The virtual color is made a subset of the physical color in order to 4210Sstevel@tonic-gate * in minimize virtual cache flushing. 4220Sstevel@tonic-gate * We add in the as to spread out different as. This happens when we 4230Sstevel@tonic-gate * initialize the start count value. 4240Sstevel@tonic-gate * sizeof(struct as) is 60 so we shift by 3 to get into the bit range 4250Sstevel@tonic-gate * that will tend to change. For example, on spitfire based machines 4260Sstevel@tonic-gate * (vcshft == 1) contigous as are spread bu ~6 bins. 4270Sstevel@tonic-gate * vcshft provides for proper virtual color alignment. 4280Sstevel@tonic-gate * In theory cnt should be updated using cas only but if we are off by one 4290Sstevel@tonic-gate * or 2 it is no big deal. 4300Sstevel@tonic-gate * We also keep a start value which is used to randomize on what bin we 4310Sstevel@tonic-gate * start counting when it is time to start another loop. This avoids 4320Sstevel@tonic-gate * contigous allocations of ecache size to point to the same bin. 4330Sstevel@tonic-gate * Why 3? Seems work ok. Better than 7 or anything larger. 4340Sstevel@tonic-gate */ 4350Sstevel@tonic-gate #define PGCLR_LOOPFACTOR 3 4360Sstevel@tonic-gate 4370Sstevel@tonic-gate /* 4380Sstevel@tonic-gate * When a bin is empty, and we can't satisfy a color request correctly, 4390Sstevel@tonic-gate * we scan. If we assume that the programs have reasonable spatial 4400Sstevel@tonic-gate * behavior, then it will not be a good idea to use the adjacent color. 4410Sstevel@tonic-gate * Using the adjacent color would result in virtually adjacent addresses 4420Sstevel@tonic-gate * mapping into the same spot in the cache. So, if we stumble across 4430Sstevel@tonic-gate * an empty bin, skip a bunch before looking. After the first skip, 4440Sstevel@tonic-gate * then just look one bin at a time so we don't miss our cache on 4450Sstevel@tonic-gate * every look. Be sure to check every bin. Page_create() will panic 4460Sstevel@tonic-gate * if we miss a page. 4470Sstevel@tonic-gate * 4480Sstevel@tonic-gate * This also explains the `<=' in the for loops in both page_get_freelist() 4490Sstevel@tonic-gate * and page_get_cachelist(). Since we checked the target bin, skipped 4500Sstevel@tonic-gate * a bunch, then continued one a time, we wind up checking the target bin 4510Sstevel@tonic-gate * twice to make sure we get all of them bins. 4520Sstevel@tonic-gate */ 4530Sstevel@tonic-gate #define BIN_STEP 20 4540Sstevel@tonic-gate 4550Sstevel@tonic-gate #ifdef VM_STATS 4560Sstevel@tonic-gate struct vmm_vmstats_str { 457414Skchow ulong_t pgf_alloc[MMU_PAGE_SIZES]; /* page_get_freelist */ 4580Sstevel@tonic-gate ulong_t pgf_allocok[MMU_PAGE_SIZES]; 4590Sstevel@tonic-gate ulong_t pgf_allocokrem[MMU_PAGE_SIZES]; 4600Sstevel@tonic-gate ulong_t pgf_allocfailed[MMU_PAGE_SIZES]; 4610Sstevel@tonic-gate ulong_t pgf_allocdeferred; 4620Sstevel@tonic-gate ulong_t pgf_allocretry[MMU_PAGE_SIZES]; 463414Skchow ulong_t pgc_alloc; /* page_get_cachelist */ 4640Sstevel@tonic-gate ulong_t pgc_allocok; 4650Sstevel@tonic-gate ulong_t pgc_allocokrem; 4660Sstevel@tonic-gate ulong_t pgc_allocokdeferred; 4670Sstevel@tonic-gate ulong_t pgc_allocfailed; 468414Skchow ulong_t pgcp_alloc[MMU_PAGE_SIZES]; /* page_get_contig_pages */ 4690Sstevel@tonic-gate ulong_t pgcp_allocfailed[MMU_PAGE_SIZES]; 4700Sstevel@tonic-gate ulong_t pgcp_allocempty[MMU_PAGE_SIZES]; 4710Sstevel@tonic-gate ulong_t pgcp_allocok[MMU_PAGE_SIZES]; 472414Skchow ulong_t ptcp[MMU_PAGE_SIZES]; /* page_trylock_contig_pages */ 4730Sstevel@tonic-gate ulong_t ptcpfreethresh[MMU_PAGE_SIZES]; 4740Sstevel@tonic-gate ulong_t ptcpfailexcl[MMU_PAGE_SIZES]; 4750Sstevel@tonic-gate ulong_t ptcpfailszc[MMU_PAGE_SIZES]; 4760Sstevel@tonic-gate ulong_t ptcpfailcage[MMU_PAGE_SIZES]; 4770Sstevel@tonic-gate ulong_t ptcpok[MMU_PAGE_SIZES]; 478414Skchow ulong_t pgmf_alloc[MMU_PAGE_SIZES]; /* page_get_mnode_freelist */ 4790Sstevel@tonic-gate ulong_t pgmf_allocfailed[MMU_PAGE_SIZES]; 4800Sstevel@tonic-gate ulong_t pgmf_allocempty[MMU_PAGE_SIZES]; 4810Sstevel@tonic-gate ulong_t pgmf_allocok[MMU_PAGE_SIZES]; 482414Skchow ulong_t pgmc_alloc; /* page_get_mnode_cachelist */ 4830Sstevel@tonic-gate ulong_t pgmc_allocfailed; 4840Sstevel@tonic-gate ulong_t pgmc_allocempty; 4850Sstevel@tonic-gate ulong_t pgmc_allocok; 486414Skchow ulong_t pladd_free[MMU_PAGE_SIZES]; /* page_list_add/sub */ 487414Skchow ulong_t plsub_free[MMU_PAGE_SIZES]; 488414Skchow ulong_t pladd_cache; 489414Skchow ulong_t plsub_cache; 490414Skchow ulong_t plsubpages_szcbig; 491414Skchow ulong_t plsubpages_szc0; 492414Skchow ulong_t pff_req[MMU_PAGE_SIZES]; /* page_freelist_fill */ 493414Skchow ulong_t pff_demote[MMU_PAGE_SIZES]; 494414Skchow ulong_t pff_coalok[MMU_PAGE_SIZES]; 495414Skchow ulong_t ppr_reloc[MMU_PAGE_SIZES]; /* page_relocate */ 4960Sstevel@tonic-gate ulong_t ppr_relocok[MMU_PAGE_SIZES]; 4970Sstevel@tonic-gate ulong_t ppr_relocnoroot[MMU_PAGE_SIZES]; 4980Sstevel@tonic-gate ulong_t ppr_reloc_replnoroot[MMU_PAGE_SIZES]; 4990Sstevel@tonic-gate ulong_t ppr_relocnolock[MMU_PAGE_SIZES]; 5000Sstevel@tonic-gate ulong_t ppr_relocnomem[MMU_PAGE_SIZES]; 5010Sstevel@tonic-gate ulong_t ppr_krelocfail[MMU_PAGE_SIZES]; 5020Sstevel@tonic-gate ulong_t page_ctrs_coalesce; /* page coalesce counter */ 5030Sstevel@tonic-gate ulong_t page_ctrs_cands_skip; /* candidates useful */ 5040Sstevel@tonic-gate ulong_t page_ctrs_changed; /* ctrs changed after locking */ 5050Sstevel@tonic-gate ulong_t page_ctrs_failed; /* page_freelist_coalesce failed */ 5060Sstevel@tonic-gate ulong_t page_ctrs_coalesce_all; /* page coalesce all counter */ 5070Sstevel@tonic-gate ulong_t page_ctrs_cands_skip_all; /* candidates useful for all func */ 5080Sstevel@tonic-gate }; 5090Sstevel@tonic-gate extern struct vmm_vmstats_str vmm_vmstats; 5100Sstevel@tonic-gate #endif /* VM_STATS */ 5110Sstevel@tonic-gate 5120Sstevel@tonic-gate /* 5130Sstevel@tonic-gate * Used to hold off page relocations into the cage until OBP has completed 5140Sstevel@tonic-gate * its boot-time handoff of its resources to the kernel. 5150Sstevel@tonic-gate */ 5160Sstevel@tonic-gate extern int page_relocate_ready; 5170Sstevel@tonic-gate 5180Sstevel@tonic-gate /* 5190Sstevel@tonic-gate * cpu/mmu-dependent vm variables may be reset at bootup. 5200Sstevel@tonic-gate */ 5210Sstevel@tonic-gate extern uint_t mmu_page_sizes; 5220Sstevel@tonic-gate extern uint_t max_mmu_page_sizes; 5230Sstevel@tonic-gate extern uint_t mmu_hashcnt; 5240Sstevel@tonic-gate extern uint_t max_mmu_hashcnt; 5250Sstevel@tonic-gate extern size_t mmu_ism_pagesize; 5260Sstevel@tonic-gate extern int mmu_exported_pagesize_mask; 5270Sstevel@tonic-gate extern uint_t mmu_exported_page_sizes; 5280Sstevel@tonic-gate extern uint_t szc_2_userszc[]; 5290Sstevel@tonic-gate extern uint_t userszc_2_szc[]; 5300Sstevel@tonic-gate 5310Sstevel@tonic-gate #define USERSZC_2_SZC(userszc) (userszc_2_szc[userszc]) 5320Sstevel@tonic-gate #define SZC_2_USERSZC(szc) (szc_2_userszc[szc]) 5330Sstevel@tonic-gate 5340Sstevel@tonic-gate /* 5350Sstevel@tonic-gate * Platform specific map_pgsz large page hook routines. 5360Sstevel@tonic-gate */ 5370Sstevel@tonic-gate extern size_t map_pgszva(struct proc *p, caddr_t addr, size_t len); 5380Sstevel@tonic-gate extern size_t map_pgszheap(struct proc *p, caddr_t addr, size_t len); 5390Sstevel@tonic-gate extern size_t map_pgszstk(struct proc *p, caddr_t addr, size_t len); 5400Sstevel@tonic-gate 5410Sstevel@tonic-gate /* 5420Sstevel@tonic-gate * Platform specific page routines 5430Sstevel@tonic-gate */ 5440Sstevel@tonic-gate extern void mach_page_add(page_t **, page_t *); 5450Sstevel@tonic-gate extern void mach_page_sub(page_t **, page_t *); 5460Sstevel@tonic-gate extern uint_t page_get_pagecolors(uint_t); 5470Sstevel@tonic-gate extern void ppcopy_kernel__relocatable(page_t *, page_t *); 5480Sstevel@tonic-gate #define ppcopy_kernel(p1, p2) ppcopy_kernel__relocatable(p1, p2) 5490Sstevel@tonic-gate 5500Sstevel@tonic-gate /* 5510Sstevel@tonic-gate * platform specific large pages for kernel heap support 5520Sstevel@tonic-gate */ 5530Sstevel@tonic-gate extern size_t get_segkmem_lpsize(size_t lpsize); 5540Sstevel@tonic-gate extern size_t mmu_get_kernel_lpsize(size_t lpsize); 5550Sstevel@tonic-gate extern void mmu_init_kernel_pgsz(struct hat *hat); 5560Sstevel@tonic-gate extern void mmu_init_kcontext(); 5570Sstevel@tonic-gate extern uint64_t kcontextreg; 5580Sstevel@tonic-gate 5590Sstevel@tonic-gate #ifdef __cplusplus 5600Sstevel@tonic-gate } 5610Sstevel@tonic-gate #endif 5620Sstevel@tonic-gate 5630Sstevel@tonic-gate #endif /* _VM_DEP_H */ 564