xref: /onnv-gate/usr/src/uts/sun4/vm/vm_dep.h (revision 1385:b13e2a3ab3e9)
10Sstevel@tonic-gate /*
20Sstevel@tonic-gate  * CDDL HEADER START
30Sstevel@tonic-gate  *
40Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
50Sstevel@tonic-gate  * Common Development and Distribution License, Version 1.0 only
60Sstevel@tonic-gate  * (the "License").  You may not use this file except in compliance
70Sstevel@tonic-gate  * with the License.
80Sstevel@tonic-gate  *
90Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
100Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
110Sstevel@tonic-gate  * See the License for the specific language governing permissions
120Sstevel@tonic-gate  * and limitations under the License.
130Sstevel@tonic-gate  *
140Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
150Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
160Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
170Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
180Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
190Sstevel@tonic-gate  *
200Sstevel@tonic-gate  * CDDL HEADER END
210Sstevel@tonic-gate  */
220Sstevel@tonic-gate /*
231373Skchow  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
240Sstevel@tonic-gate  * Use is subject to license terms.
250Sstevel@tonic-gate  */
260Sstevel@tonic-gate 
270Sstevel@tonic-gate /*
280Sstevel@tonic-gate  * UNIX machine dependent virtual memory support.
290Sstevel@tonic-gate  */
300Sstevel@tonic-gate 
310Sstevel@tonic-gate #ifndef	_VM_DEP_H
320Sstevel@tonic-gate #define	_VM_DEP_H
330Sstevel@tonic-gate 
340Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
350Sstevel@tonic-gate 
360Sstevel@tonic-gate #ifdef	__cplusplus
370Sstevel@tonic-gate extern "C" {
380Sstevel@tonic-gate #endif
390Sstevel@tonic-gate 
400Sstevel@tonic-gate #include <vm/hat_sfmmu.h>
410Sstevel@tonic-gate #include <sys/archsystm.h>
420Sstevel@tonic-gate #include <sys/memnode.h>
430Sstevel@tonic-gate 
440Sstevel@tonic-gate #define	GETTICK()	gettick()
450Sstevel@tonic-gate 
460Sstevel@tonic-gate /*
470Sstevel@tonic-gate  * Per page size free lists. Allocated dynamically.
480Sstevel@tonic-gate  */
490Sstevel@tonic-gate #define	MAX_MEM_TYPES	2	/* 0 = reloc, 1 = noreloc */
500Sstevel@tonic-gate #define	MTYPE_RELOC	0
510Sstevel@tonic-gate #define	MTYPE_NORELOC	1
520Sstevel@tonic-gate 
530Sstevel@tonic-gate #define	PP_2_MTYPE(pp)	(PP_ISNORELOC(pp) ? MTYPE_NORELOC : MTYPE_RELOC)
540Sstevel@tonic-gate 
55*1385Skchow #define	MTYPE_INIT(mtype, vp, vaddr, flags, pgsz)			\
560Sstevel@tonic-gate 	mtype = (flags & PG_NORELOC) ? MTYPE_NORELOC : MTYPE_RELOC;
570Sstevel@tonic-gate 
580Sstevel@tonic-gate /* mtype init for page_get_replacement_page */
590Sstevel@tonic-gate 
60*1385Skchow #define	MTYPE_PGR_INIT(mtype, flags, pp, mnode, pgcnt)			\
610Sstevel@tonic-gate 	mtype = (flags & PG_NORELOC) ? MTYPE_NORELOC : MTYPE_RELOC;
620Sstevel@tonic-gate 
630Sstevel@tonic-gate #define	MNODETYPE_2_PFN(mnode, mtype, pfnlo, pfnhi)			\
640Sstevel@tonic-gate 	ASSERT(mtype != MTYPE_NORELOC);					\
650Sstevel@tonic-gate 	pfnlo = mem_node_config[mnode].physbase;			\
660Sstevel@tonic-gate 	pfnhi = mem_node_config[mnode].physmax;
670Sstevel@tonic-gate 
680Sstevel@tonic-gate /*
690Sstevel@tonic-gate  * Internal PG_ flags.
700Sstevel@tonic-gate  */
710Sstevel@tonic-gate #define	PGI_RELOCONLY	0x10000	/* acts in the opposite sense to PG_NORELOC */
720Sstevel@tonic-gate #define	PGI_NOCAGE	0x20000	/* indicates Cage is disabled */
730Sstevel@tonic-gate #define	PGI_PGCPHIPRI	0x40000	/* page_get_contig_page priority allocation */
740Sstevel@tonic-gate #define	PGI_PGCPSZC0	0x80000	/* relocate base pagesize page */
750Sstevel@tonic-gate 
760Sstevel@tonic-gate /*
770Sstevel@tonic-gate  * PGI mtype flags - should not overlap PGI flags
780Sstevel@tonic-gate  */
790Sstevel@tonic-gate #define	PGI_MT_RANGE	0x1000000	/* mtype range */
800Sstevel@tonic-gate #define	PGI_MT_NEXT	0x2000000	/* get next mtype */
810Sstevel@tonic-gate 
820Sstevel@tonic-gate extern page_t ***page_freelists[MMU_PAGE_SIZES][MAX_MEM_TYPES];
830Sstevel@tonic-gate extern page_t ***page_cachelists[MAX_MEM_TYPES];
840Sstevel@tonic-gate 
850Sstevel@tonic-gate #define	PAGE_FREELISTS(mnode, szc, color, mtype) \
860Sstevel@tonic-gate 	(*(page_freelists[szc][mtype][mnode] + (color)))
870Sstevel@tonic-gate 
880Sstevel@tonic-gate #define	PAGE_CACHELISTS(mnode, color, mtype) \
890Sstevel@tonic-gate 	(*(page_cachelists[mtype][mnode] + (color)))
900Sstevel@tonic-gate 
910Sstevel@tonic-gate /*
920Sstevel@tonic-gate  * There are 'page_colors' colors/bins.  Spread them out under a
930Sstevel@tonic-gate  * couple of locks.  There are mutexes for both the page freelist
940Sstevel@tonic-gate  * and the page cachelist.  We want enough locks to make contention
950Sstevel@tonic-gate  * reasonable, but not too many -- otherwise page_freelist_lock() gets
960Sstevel@tonic-gate  * so expensive that it becomes the bottleneck!
970Sstevel@tonic-gate  */
980Sstevel@tonic-gate #define	NPC_MUTEX	16
990Sstevel@tonic-gate 
1000Sstevel@tonic-gate extern kmutex_t	*fpc_mutex[NPC_MUTEX];
1010Sstevel@tonic-gate extern kmutex_t	*cpc_mutex[NPC_MUTEX];
1020Sstevel@tonic-gate 
1030Sstevel@tonic-gate /* Find the bin for the given page if it was of size szc */
1040Sstevel@tonic-gate #define	PP_2_BIN_SZC(pp, szc)                                           \
1050Sstevel@tonic-gate 	(((pp->p_pagenum) & page_colors_mask) >>                        \
1060Sstevel@tonic-gate 	(hw_page_array[szc].hp_shift - hw_page_array[0].hp_shift))
1070Sstevel@tonic-gate 
1080Sstevel@tonic-gate #define	PP_2_BIN(pp)		(PP_2_BIN_SZC(pp, pp->p_szc))
1090Sstevel@tonic-gate 
1100Sstevel@tonic-gate #define	PP_2_MEM_NODE(pp)	(PFN_2_MEM_NODE(pp->p_pagenum))
1110Sstevel@tonic-gate 
1120Sstevel@tonic-gate #define	PC_BIN_MUTEX(mnode, bin, flags) ((flags & PG_FREE_LIST) ?	\
1130Sstevel@tonic-gate 	&fpc_mutex[(bin) & (NPC_MUTEX - 1)][mnode] :			\
1140Sstevel@tonic-gate 	&cpc_mutex[(bin) & (NPC_MUTEX - 1)][mnode])
1150Sstevel@tonic-gate 
1160Sstevel@tonic-gate #define	FPC_MUTEX(mnode, i)	(&fpc_mutex[i][mnode])
1170Sstevel@tonic-gate #define	CPC_MUTEX(mnode, i)	(&cpc_mutex[i][mnode])
1180Sstevel@tonic-gate 
1190Sstevel@tonic-gate #define	PFN_BASE(pfnum, szc)	(pfnum & ~((1 << PAGE_BSZS_SHIFT(szc)) - 1))
1200Sstevel@tonic-gate 
1210Sstevel@tonic-gate typedef	char	hpmctr_t;
1220Sstevel@tonic-gate 
1230Sstevel@tonic-gate #ifdef DEBUG
1240Sstevel@tonic-gate #define	CHK_LPG(pp, szc)	chk_lpg(pp, szc)
1250Sstevel@tonic-gate extern void	chk_lpg(page_t *, uchar_t);
1260Sstevel@tonic-gate #else
1270Sstevel@tonic-gate #define	CHK_LPG(pp, szc)
1280Sstevel@tonic-gate #endif
1290Sstevel@tonic-gate 
130414Skchow /*
131414Skchow  * page list count per mnode and type.
132414Skchow  */
1330Sstevel@tonic-gate typedef	struct {
134414Skchow 	pgcnt_t	plc_mt_pgmax;		/* max page cnt */
135414Skchow 	pgcnt_t plc_mt_clpgcnt;		/* cache list cnt */
136414Skchow 	pgcnt_t plc_mt_flpgcnt;		/* free list cnt - small pages */
137414Skchow 	pgcnt_t plc_mt_lgpgcnt;		/* free list cnt - large pages */
138414Skchow #ifdef DEBUG
1390Sstevel@tonic-gate 	struct {
140414Skchow 		pgcnt_t plc_mts_pgcnt;	/* per page size count */
141414Skchow 		int	plc_mts_colors;
142414Skchow 		pgcnt_t	*plc_mtsc_pgcnt; /* per color bin count */
143414Skchow 	} plc_mts[MMU_PAGE_SIZES];
144414Skchow #endif
145414Skchow } plcnt_t[MAX_MEM_NODES][MAX_MEM_TYPES];
1460Sstevel@tonic-gate 
147414Skchow #ifdef DEBUG
1480Sstevel@tonic-gate 
1490Sstevel@tonic-gate #define	PLCNT_SZ(ctrs_sz) {						\
1500Sstevel@tonic-gate 	int	szc;							\
1510Sstevel@tonic-gate 	for (szc = 0; szc <= mmu_page_sizes; szc++) {			\
1520Sstevel@tonic-gate 		int	colors = page_get_pagecolors(szc);		\
1530Sstevel@tonic-gate 		ctrs_sz += (max_mem_nodes * MAX_MEM_TYPES *		\
1540Sstevel@tonic-gate 		    colors * sizeof (pgcnt_t));				\
1550Sstevel@tonic-gate 	}								\
1560Sstevel@tonic-gate }
1570Sstevel@tonic-gate 
1580Sstevel@tonic-gate #define	PLCNT_INIT(base) {						\
1590Sstevel@tonic-gate 	int	mn, mt, szc, colors;					\
1600Sstevel@tonic-gate 	for (szc = 0; szc < mmu_page_sizes; szc++) {			\
1610Sstevel@tonic-gate 		colors = page_get_pagecolors(szc);			\
1620Sstevel@tonic-gate 		for (mn = 0; mn < max_mem_nodes; mn++) {		\
1630Sstevel@tonic-gate 			for (mt = 0; mt < MAX_MEM_TYPES; mt++) {	\
164*1385Skchow 				plcnt[mn][mt].plc_mts[szc].		\
1650Sstevel@tonic-gate 				    plc_mts_colors = colors;		\
166*1385Skchow 				plcnt[mn][mt].plc_mts[szc].		\
1670Sstevel@tonic-gate 				    plc_mtsc_pgcnt = (pgcnt_t *)base;	\
1680Sstevel@tonic-gate 				base += (colors * sizeof (pgcnt_t));	\
1690Sstevel@tonic-gate 			}						\
1700Sstevel@tonic-gate 		}							\
1710Sstevel@tonic-gate 	}								\
1720Sstevel@tonic-gate }
1730Sstevel@tonic-gate 
174414Skchow #define	PLCNT_DO(pp, mn, mtype, szc, cnt, flags) {			\
1750Sstevel@tonic-gate 	int	bin = PP_2_BIN(pp);					\
1760Sstevel@tonic-gate 	if (flags & PG_CACHE_LIST)					\
177414Skchow 		atomic_add_long(&plcnt[mn][mtype].plc_mt_clpgcnt, cnt);	\
178414Skchow 	else if (szc)							\
179414Skchow 		atomic_add_long(&plcnt[mn][mtype].plc_mt_lgpgcnt, cnt);	\
180414Skchow 	else								\
181414Skchow 		atomic_add_long(&plcnt[mn][mtype].plc_mt_flpgcnt, cnt);	\
182414Skchow 	atomic_add_long(&plcnt[mn][mtype].plc_mts[szc].plc_mts_pgcnt,	\
183414Skchow 	    cnt);							\
184414Skchow 	atomic_add_long(&plcnt[mn][mtype].plc_mts[szc].			\
1850Sstevel@tonic-gate 	    plc_mtsc_pgcnt[bin], cnt);					\
1860Sstevel@tonic-gate }
1870Sstevel@tonic-gate 
1880Sstevel@tonic-gate #else
1890Sstevel@tonic-gate 
1900Sstevel@tonic-gate #define	PLCNT_SZ(ctrs_sz)
1910Sstevel@tonic-gate 
1920Sstevel@tonic-gate #define	PLCNT_INIT(base)
1930Sstevel@tonic-gate 
194414Skchow /* PG_FREE_LIST may not be explicitly set in flags for large pages */
1950Sstevel@tonic-gate 
196414Skchow #define	PLCNT_DO(pp, mn, mtype, szc, cnt, flags) {			\
197414Skchow 	if (flags & PG_CACHE_LIST)					\
198414Skchow 		atomic_add_long(&plcnt[mn][mtype].plc_mt_clpgcnt, cnt);	\
199414Skchow 	else if (szc)							\
200414Skchow 		atomic_add_long(&plcnt[mn][mtype].plc_mt_lgpgcnt, cnt);	\
201414Skchow 	else								\
202414Skchow 		atomic_add_long(&plcnt[mn][mtype].plc_mt_flpgcnt, cnt);	\
2030Sstevel@tonic-gate }
2040Sstevel@tonic-gate 
2050Sstevel@tonic-gate #endif
2060Sstevel@tonic-gate 
207414Skchow #define	PLCNT_INCR(pp, mn, mtype, szc, flags) {				\
208414Skchow 	long	cnt = (1 << PAGE_BSZS_SHIFT(szc));			\
209414Skchow 	PLCNT_DO(pp, mn, mtype, szc, cnt, flags);			\
210414Skchow }
211414Skchow 
212414Skchow #define	PLCNT_DECR(pp, mn, mtype, szc, flags) {				\
213414Skchow 	long	cnt = ((-1) << PAGE_BSZS_SHIFT(szc));			\
214414Skchow 	PLCNT_DO(pp, mn, mtype, szc, cnt, flags);			\
215414Skchow }
216414Skchow 
217414Skchow /*
218414Skchow  * macros to update page list max counts - done when pages transferred
2191373Skchow  * from RELOC to NORELOC mtype (kcage_init or kcage_assimilate_page).
220414Skchow  */
2211373Skchow 
2221373Skchow #define	PLCNT_XFER_NORELOC(pp) {					\
2231373Skchow 	long	cnt = (1 << PAGE_BSZS_SHIFT((pp)->p_szc));		\
2241373Skchow 	int	mn = PP_2_MEM_NODE(pp);					\
2251373Skchow 	atomic_add_long(&plcnt[mn][MTYPE_NORELOC].plc_mt_pgmax, cnt);	\
2261373Skchow 	atomic_add_long(&plcnt[mn][MTYPE_RELOC].plc_mt_pgmax, -cnt);	\
227414Skchow }
228414Skchow 
2291373Skchow /*
2301373Skchow  * macro to modify the page list max counts when memory is added to
2311373Skchow  * the page lists during startup (add_physmem) or during a DR operation
2321373Skchow  * when memory is added (kphysm_add_memory_dynamic) or deleted
2331373Skchow  * (kphysm_del_cleanup).
2341373Skchow  */
2351373Skchow #define	PLCNT_MODIFY_MAX(pfn, cnt) {					\
2361373Skchow 	int	mn = PFN_2_MEM_NODE(pfn);				\
2371373Skchow 	atomic_add_long(&plcnt[mn][MTYPE_RELOC].plc_mt_pgmax, (cnt));	\
238414Skchow }
239414Skchow 
240414Skchow extern plcnt_t	plcnt;
241414Skchow 
242414Skchow #define	MNODE_PGCNT(mn)							\
243414Skchow 	(plcnt[mn][MTYPE_RELOC].plc_mt_clpgcnt +			\
244414Skchow 	    plcnt[mn][MTYPE_NORELOC].plc_mt_clpgcnt +			\
245414Skchow 	    plcnt[mn][MTYPE_RELOC].plc_mt_flpgcnt +			\
246414Skchow 	    plcnt[mn][MTYPE_NORELOC].plc_mt_flpgcnt +			\
247414Skchow 	    plcnt[mn][MTYPE_RELOC].plc_mt_lgpgcnt +			\
248414Skchow 	    plcnt[mn][MTYPE_NORELOC].plc_mt_lgpgcnt)
249414Skchow 
250414Skchow #define	MNODETYPE_PGCNT(mn, mtype)					\
251414Skchow 	(plcnt[mn][mtype].plc_mt_clpgcnt +				\
252414Skchow 	    plcnt[mn][mtype].plc_mt_flpgcnt +				\
253414Skchow 	    plcnt[mn][mtype].plc_mt_lgpgcnt)
254414Skchow 
255414Skchow /*
256414Skchow  * macros to loop through the mtype range - MTYPE_START returns -1 in
257414Skchow  * mtype if no pages in mnode/mtype and possibly NEXT mtype.
258414Skchow  */
259414Skchow #define	MTYPE_START(mnode, mtype, flags) {				\
260414Skchow 	if (plcnt[mnode][mtype].plc_mt_pgmax == 0) {			\
261414Skchow 		ASSERT(MNODETYPE_PGCNT(mnode, mtype) == 0);		\
262414Skchow 		MTYPE_NEXT(mnode, mtype, flags);			\
263414Skchow 	}								\
264414Skchow }
265414Skchow 
266414Skchow /*
267414Skchow  * if allocation from the RELOC pool failed and there is sufficient cage
268414Skchow  * memory, attempt to allocate from the NORELOC pool.
269414Skchow  */
270414Skchow #define	MTYPE_NEXT(mnode, mtype, flags) { 				\
271414Skchow 	if (!(flags & (PG_NORELOC | PGI_NOCAGE | PGI_RELOCONLY)) &&	\
272414Skchow 	    (kcage_freemem >= kcage_lotsfree)) {			\
273414Skchow 		if (plcnt[mnode][mtype].plc_mt_pgmax == 0) {		\
274414Skchow 			ASSERT(MNODETYPE_PGCNT(mnode, mtype) == 0);	\
275414Skchow 			mtype = -1;					\
276414Skchow 		} else {						\
277414Skchow 			mtype = MTYPE_NORELOC;				\
278414Skchow 			flags |= PG_NORELOC;				\
279414Skchow 		}							\
280414Skchow 	} else {							\
281414Skchow 		mtype = -1;						\
282414Skchow 	}								\
283414Skchow }
284414Skchow 
2850Sstevel@tonic-gate /*
2860Sstevel@tonic-gate  * get the ecache setsize for the current cpu.
2870Sstevel@tonic-gate  */
2880Sstevel@tonic-gate #define	CPUSETSIZE()	(cpunodes[CPU->cpu_id].ecache_setsize)
2890Sstevel@tonic-gate 
290414Skchow extern struct cpu	cpu0;
291414Skchow #define	CPU0		&cpu0
2920Sstevel@tonic-gate 
2930Sstevel@tonic-gate #define	PAGE_BSZS_SHIFT(szc)	TTE_BSZS_SHIFT(szc)
2940Sstevel@tonic-gate /*
2950Sstevel@tonic-gate  * For sfmmu each larger page is 8 times the size of the previous
2960Sstevel@tonic-gate  * size page.
2970Sstevel@tonic-gate  */
2980Sstevel@tonic-gate #define	FULL_REGION_CNT(rg_szc)	(8)
2990Sstevel@tonic-gate 
3000Sstevel@tonic-gate /*
3010Sstevel@tonic-gate  * The counter base must be per page_counter element to prevent
3020Sstevel@tonic-gate  * races when re-indexing, and the base page size element should
3030Sstevel@tonic-gate  * be aligned on a boundary of the given region size.
3040Sstevel@tonic-gate  *
3050Sstevel@tonic-gate  * We also round up the number of pages spanned by the counters
3060Sstevel@tonic-gate  * for a given region to PC_BASE_ALIGN in certain situations to simplify
3070Sstevel@tonic-gate  * the coding for some non-performance critical routines.
3080Sstevel@tonic-gate  */
3090Sstevel@tonic-gate #define	PC_BASE_ALIGN		((pfn_t)1 << PAGE_BSZS_SHIFT(mmu_page_sizes-1))
3100Sstevel@tonic-gate #define	PC_BASE_ALIGN_MASK	(PC_BASE_ALIGN - 1)
3110Sstevel@tonic-gate 
3120Sstevel@tonic-gate extern int ecache_alignsize;
3130Sstevel@tonic-gate #define	L2CACHE_ALIGN		ecache_alignsize
314450Skchow #define	L2CACHE_ALIGN_MAX	512
3150Sstevel@tonic-gate 
3160Sstevel@tonic-gate extern int consistent_coloring;
3170Sstevel@tonic-gate extern uint_t vac_colors_mask;
3180Sstevel@tonic-gate extern int vac_size;
3190Sstevel@tonic-gate extern int vac_shift;
3200Sstevel@tonic-gate 
3210Sstevel@tonic-gate /*
3220Sstevel@tonic-gate  * Auto large page selection support variables. Some CPU
3230Sstevel@tonic-gate  * implementations may differ from the defaults and will need
3240Sstevel@tonic-gate  * to change these.
3250Sstevel@tonic-gate  */
3260Sstevel@tonic-gate extern int auto_lpg_tlb_threshold;
3270Sstevel@tonic-gate extern int auto_lpg_minszc;
3280Sstevel@tonic-gate extern int auto_lpg_maxszc;
3290Sstevel@tonic-gate extern size_t auto_lpg_heap_default;
3300Sstevel@tonic-gate extern size_t auto_lpg_stack_default;
3310Sstevel@tonic-gate extern size_t auto_lpg_va_default;
3320Sstevel@tonic-gate extern size_t auto_lpg_remap_threshold;
333423Sdavemq extern pgcnt_t auto_lpg_min_physmem;
3340Sstevel@tonic-gate 
3350Sstevel@tonic-gate /*
3360Sstevel@tonic-gate  * AS_2_BIN macro controls the page coloring policy.
3370Sstevel@tonic-gate  * 0 (default) uses various vaddr bits
3380Sstevel@tonic-gate  * 1 virtual=paddr
3390Sstevel@tonic-gate  * 2 bin hopping
3400Sstevel@tonic-gate  */
3410Sstevel@tonic-gate #define	AS_2_BIN(as, seg, vp, addr, bin)				\
3420Sstevel@tonic-gate switch (consistent_coloring) {						\
3430Sstevel@tonic-gate 	default:                                                        \
3440Sstevel@tonic-gate 		cmn_err(CE_WARN,					\
3450Sstevel@tonic-gate 			"AS_2_BIN: bad consistent coloring value");	\
3460Sstevel@tonic-gate 		/* assume default algorithm -> continue */		\
3470Sstevel@tonic-gate 	case 0: {                                                       \
3480Sstevel@tonic-gate 		uint32_t ndx, new;					\
3490Sstevel@tonic-gate 		int slew = 0;						\
3500Sstevel@tonic-gate                                                                         \
3510Sstevel@tonic-gate 		if (vp != NULL && IS_SWAPVP(vp) &&			\
3520Sstevel@tonic-gate 			seg->s_ops == &segvn_ops)			\
3530Sstevel@tonic-gate 			slew = as_color_bin(as);			\
3540Sstevel@tonic-gate                                                                         \
3550Sstevel@tonic-gate 		bin = (((uintptr_t)addr >> MMU_PAGESHIFT) +		\
3560Sstevel@tonic-gate 			(((uintptr_t)addr >> page_coloring_shift) <<	\
3570Sstevel@tonic-gate 			(vac_shift - MMU_PAGESHIFT)) + slew) &		\
3580Sstevel@tonic-gate 			page_colors_mask;				\
3590Sstevel@tonic-gate                                                                         \
3600Sstevel@tonic-gate 		break;                                                  \
3610Sstevel@tonic-gate 	}                                                               \
3620Sstevel@tonic-gate 	case 1:                                                         \
3630Sstevel@tonic-gate 		bin = ((uintptr_t)addr >> MMU_PAGESHIFT) &		\
3640Sstevel@tonic-gate 			page_colors_mask;				\
3650Sstevel@tonic-gate 		break;                                                  \
3660Sstevel@tonic-gate 	case 2: {                                                       \
3670Sstevel@tonic-gate 		int cnt = as_color_bin(as);				\
3680Sstevel@tonic-gate 		/* make sure physical color aligns with vac color */	\
3690Sstevel@tonic-gate 		while ((cnt & vac_colors_mask) !=			\
3700Sstevel@tonic-gate 		    addr_to_vcolor(addr)) {				\
3710Sstevel@tonic-gate 			cnt++;						\
3720Sstevel@tonic-gate 		}                                                       \
3730Sstevel@tonic-gate 		bin = cnt = cnt & page_colors_mask;			\
3740Sstevel@tonic-gate 		/* update per as page coloring fields */		\
3750Sstevel@tonic-gate 		cnt = (cnt + 1) & page_colors_mask;			\
3760Sstevel@tonic-gate 		if (cnt == (as_color_start(as) & page_colors_mask)) {	\
3770Sstevel@tonic-gate 			cnt = as_color_start(as) = as_color_start(as) + \
3780Sstevel@tonic-gate 				PGCLR_LOOPFACTOR;			\
3790Sstevel@tonic-gate 		}                                                       \
3800Sstevel@tonic-gate 		as_color_bin(as) = cnt & page_colors_mask;		\
3810Sstevel@tonic-gate 		break;                                                  \
3820Sstevel@tonic-gate 	}								\
3830Sstevel@tonic-gate }									\
3840Sstevel@tonic-gate 	ASSERT(bin <= page_colors_mask);
3850Sstevel@tonic-gate 
3860Sstevel@tonic-gate /*
387414Skchow  * cpu private vm data - accessed thru CPU->cpu_vm_data
388414Skchow  *	vc_pnum_memseg: tracks last memseg visited in page_numtopp_nolock()
389414Skchow  *	vc_pnext_memseg: tracks last memseg visited in page_nextn()
390414Skchow  *	vc_kmptr: unaligned kmem pointer for this vm_cpu_data_t
391450Skchow  *	vc_kmsize: orignal kmem size for this vm_cpu_data_t
392414Skchow  */
393414Skchow 
394414Skchow typedef struct {
395414Skchow 	struct memseg	*vc_pnum_memseg;
396414Skchow 	struct memseg	*vc_pnext_memseg;
397414Skchow 	void		*vc_kmptr;
398450Skchow 	size_t		vc_kmsize;
399414Skchow } vm_cpu_data_t;
400414Skchow 
401414Skchow /* allocation size to ensure vm_cpu_data_t resides in its own cache line */
402414Skchow #define	VM_CPU_DATA_PADSIZE						\
403414Skchow 	(P2ROUNDUP(sizeof (vm_cpu_data_t), L2CACHE_ALIGN_MAX))
404414Skchow 
405414Skchow /* for boot cpu before kmem is initialized */
406414Skchow extern char	vm_cpu_data0[];
407414Skchow 
408414Skchow /*
4090Sstevel@tonic-gate  * Function to get an ecache color bin: F(as, cnt, vcolor).
4100Sstevel@tonic-gate  * the goal of this function is to:
4110Sstevel@tonic-gate  * - to spread a processes' physical pages across the entire ecache to
4120Sstevel@tonic-gate  *	maximize its use.
4130Sstevel@tonic-gate  * - to minimize vac flushes caused when we reuse a physical page on a
4140Sstevel@tonic-gate  *	different vac color than it was previously used.
4150Sstevel@tonic-gate  * - to prevent all processes to use the same exact colors and trash each
4160Sstevel@tonic-gate  *	other.
4170Sstevel@tonic-gate  *
4180Sstevel@tonic-gate  * cnt is a bin ptr kept on a per as basis.  As we page_create we increment
4190Sstevel@tonic-gate  * the ptr so we spread out the physical pages to cover the entire ecache.
4200Sstevel@tonic-gate  * The virtual color is made a subset of the physical color in order to
4210Sstevel@tonic-gate  * in minimize virtual cache flushing.
4220Sstevel@tonic-gate  * We add in the as to spread out different as.	 This happens when we
4230Sstevel@tonic-gate  * initialize the start count value.
4240Sstevel@tonic-gate  * sizeof(struct as) is 60 so we shift by 3 to get into the bit range
4250Sstevel@tonic-gate  * that will tend to change.  For example, on spitfire based machines
4260Sstevel@tonic-gate  * (vcshft == 1) contigous as are spread bu ~6 bins.
4270Sstevel@tonic-gate  * vcshft provides for proper virtual color alignment.
4280Sstevel@tonic-gate  * In theory cnt should be updated using cas only but if we are off by one
4290Sstevel@tonic-gate  * or 2 it is no big deal.
4300Sstevel@tonic-gate  * We also keep a start value which is used to randomize on what bin we
4310Sstevel@tonic-gate  * start counting when it is time to start another loop. This avoids
4320Sstevel@tonic-gate  * contigous allocations of ecache size to point to the same bin.
4330Sstevel@tonic-gate  * Why 3? Seems work ok. Better than 7 or anything larger.
4340Sstevel@tonic-gate  */
4350Sstevel@tonic-gate #define	PGCLR_LOOPFACTOR 3
4360Sstevel@tonic-gate 
4370Sstevel@tonic-gate /*
4380Sstevel@tonic-gate  * When a bin is empty, and we can't satisfy a color request correctly,
4390Sstevel@tonic-gate  * we scan.  If we assume that the programs have reasonable spatial
4400Sstevel@tonic-gate  * behavior, then it will not be a good idea to use the adjacent color.
4410Sstevel@tonic-gate  * Using the adjacent color would result in virtually adjacent addresses
4420Sstevel@tonic-gate  * mapping into the same spot in the cache.  So, if we stumble across
4430Sstevel@tonic-gate  * an empty bin, skip a bunch before looking.  After the first skip,
4440Sstevel@tonic-gate  * then just look one bin at a time so we don't miss our cache on
4450Sstevel@tonic-gate  * every look. Be sure to check every bin.  Page_create() will panic
4460Sstevel@tonic-gate  * if we miss a page.
4470Sstevel@tonic-gate  *
4480Sstevel@tonic-gate  * This also explains the `<=' in the for loops in both page_get_freelist()
4490Sstevel@tonic-gate  * and page_get_cachelist().  Since we checked the target bin, skipped
4500Sstevel@tonic-gate  * a bunch, then continued one a time, we wind up checking the target bin
4510Sstevel@tonic-gate  * twice to make sure we get all of them bins.
4520Sstevel@tonic-gate  */
4530Sstevel@tonic-gate #define	BIN_STEP	20
4540Sstevel@tonic-gate 
4550Sstevel@tonic-gate #ifdef VM_STATS
4560Sstevel@tonic-gate struct vmm_vmstats_str {
457414Skchow 	ulong_t pgf_alloc[MMU_PAGE_SIZES];	/* page_get_freelist */
4580Sstevel@tonic-gate 	ulong_t pgf_allocok[MMU_PAGE_SIZES];
4590Sstevel@tonic-gate 	ulong_t pgf_allocokrem[MMU_PAGE_SIZES];
4600Sstevel@tonic-gate 	ulong_t pgf_allocfailed[MMU_PAGE_SIZES];
4610Sstevel@tonic-gate 	ulong_t pgf_allocdeferred;
4620Sstevel@tonic-gate 	ulong_t	pgf_allocretry[MMU_PAGE_SIZES];
463414Skchow 	ulong_t pgc_alloc;			/* page_get_cachelist */
4640Sstevel@tonic-gate 	ulong_t pgc_allocok;
4650Sstevel@tonic-gate 	ulong_t pgc_allocokrem;
4660Sstevel@tonic-gate 	ulong_t	pgc_allocokdeferred;
4670Sstevel@tonic-gate 	ulong_t pgc_allocfailed;
468414Skchow 	ulong_t	pgcp_alloc[MMU_PAGE_SIZES];	/* page_get_contig_pages */
4690Sstevel@tonic-gate 	ulong_t	pgcp_allocfailed[MMU_PAGE_SIZES];
4700Sstevel@tonic-gate 	ulong_t	pgcp_allocempty[MMU_PAGE_SIZES];
4710Sstevel@tonic-gate 	ulong_t	pgcp_allocok[MMU_PAGE_SIZES];
472414Skchow 	ulong_t	ptcp[MMU_PAGE_SIZES];		/* page_trylock_contig_pages */
4730Sstevel@tonic-gate 	ulong_t	ptcpfreethresh[MMU_PAGE_SIZES];
4740Sstevel@tonic-gate 	ulong_t	ptcpfailexcl[MMU_PAGE_SIZES];
4750Sstevel@tonic-gate 	ulong_t	ptcpfailszc[MMU_PAGE_SIZES];
4760Sstevel@tonic-gate 	ulong_t	ptcpfailcage[MMU_PAGE_SIZES];
4770Sstevel@tonic-gate 	ulong_t	ptcpok[MMU_PAGE_SIZES];
478414Skchow 	ulong_t	pgmf_alloc[MMU_PAGE_SIZES];	/* page_get_mnode_freelist */
4790Sstevel@tonic-gate 	ulong_t	pgmf_allocfailed[MMU_PAGE_SIZES];
4800Sstevel@tonic-gate 	ulong_t	pgmf_allocempty[MMU_PAGE_SIZES];
4810Sstevel@tonic-gate 	ulong_t	pgmf_allocok[MMU_PAGE_SIZES];
482414Skchow 	ulong_t	pgmc_alloc;			/* page_get_mnode_cachelist */
4830Sstevel@tonic-gate 	ulong_t	pgmc_allocfailed;
4840Sstevel@tonic-gate 	ulong_t	pgmc_allocempty;
4850Sstevel@tonic-gate 	ulong_t	pgmc_allocok;
486414Skchow 	ulong_t	pladd_free[MMU_PAGE_SIZES];	/* page_list_add/sub */
487414Skchow 	ulong_t	plsub_free[MMU_PAGE_SIZES];
488414Skchow 	ulong_t	pladd_cache;
489414Skchow 	ulong_t	plsub_cache;
490414Skchow 	ulong_t	plsubpages_szcbig;
491414Skchow 	ulong_t	plsubpages_szc0;
492414Skchow 	ulong_t	pff_req[MMU_PAGE_SIZES];	/* page_freelist_fill */
493414Skchow 	ulong_t	pff_demote[MMU_PAGE_SIZES];
494414Skchow 	ulong_t	pff_coalok[MMU_PAGE_SIZES];
495414Skchow 	ulong_t ppr_reloc[MMU_PAGE_SIZES];	/* page_relocate */
4960Sstevel@tonic-gate 	ulong_t ppr_relocok[MMU_PAGE_SIZES];
4970Sstevel@tonic-gate 	ulong_t ppr_relocnoroot[MMU_PAGE_SIZES];
4980Sstevel@tonic-gate 	ulong_t ppr_reloc_replnoroot[MMU_PAGE_SIZES];
4990Sstevel@tonic-gate 	ulong_t ppr_relocnolock[MMU_PAGE_SIZES];
5000Sstevel@tonic-gate 	ulong_t ppr_relocnomem[MMU_PAGE_SIZES];
5010Sstevel@tonic-gate 	ulong_t ppr_krelocfail[MMU_PAGE_SIZES];
5020Sstevel@tonic-gate 	ulong_t	page_ctrs_coalesce;	/* page coalesce counter */
5030Sstevel@tonic-gate 	ulong_t	page_ctrs_cands_skip;	/* candidates useful */
5040Sstevel@tonic-gate 	ulong_t	page_ctrs_changed;	/* ctrs changed after locking */
5050Sstevel@tonic-gate 	ulong_t	page_ctrs_failed;	/* page_freelist_coalesce failed */
5060Sstevel@tonic-gate 	ulong_t	page_ctrs_coalesce_all;	/* page coalesce all counter */
5070Sstevel@tonic-gate 	ulong_t	page_ctrs_cands_skip_all; /* candidates useful for all func */
5080Sstevel@tonic-gate };
5090Sstevel@tonic-gate extern struct vmm_vmstats_str vmm_vmstats;
5100Sstevel@tonic-gate #endif	/* VM_STATS */
5110Sstevel@tonic-gate 
5120Sstevel@tonic-gate /*
5130Sstevel@tonic-gate  * Used to hold off page relocations into the cage until OBP has completed
5140Sstevel@tonic-gate  * its boot-time handoff of its resources to the kernel.
5150Sstevel@tonic-gate  */
5160Sstevel@tonic-gate extern int page_relocate_ready;
5170Sstevel@tonic-gate 
5180Sstevel@tonic-gate /*
5190Sstevel@tonic-gate  * cpu/mmu-dependent vm variables may be reset at bootup.
5200Sstevel@tonic-gate  */
5210Sstevel@tonic-gate extern uint_t mmu_page_sizes;
5220Sstevel@tonic-gate extern uint_t max_mmu_page_sizes;
5230Sstevel@tonic-gate extern uint_t mmu_hashcnt;
5240Sstevel@tonic-gate extern uint_t max_mmu_hashcnt;
5250Sstevel@tonic-gate extern size_t mmu_ism_pagesize;
5260Sstevel@tonic-gate extern int mmu_exported_pagesize_mask;
5270Sstevel@tonic-gate extern uint_t mmu_exported_page_sizes;
5280Sstevel@tonic-gate extern uint_t szc_2_userszc[];
5290Sstevel@tonic-gate extern uint_t userszc_2_szc[];
5300Sstevel@tonic-gate 
5310Sstevel@tonic-gate #define	USERSZC_2_SZC(userszc)	(userszc_2_szc[userszc])
5320Sstevel@tonic-gate #define	SZC_2_USERSZC(szc)	(szc_2_userszc[szc])
5330Sstevel@tonic-gate 
5340Sstevel@tonic-gate /*
5350Sstevel@tonic-gate  * Platform specific map_pgsz large page hook routines.
5360Sstevel@tonic-gate  */
5370Sstevel@tonic-gate extern size_t map_pgszva(struct proc *p, caddr_t addr, size_t len);
5380Sstevel@tonic-gate extern size_t map_pgszheap(struct proc *p, caddr_t addr, size_t len);
5390Sstevel@tonic-gate extern size_t map_pgszstk(struct proc *p, caddr_t addr, size_t len);
5400Sstevel@tonic-gate 
5410Sstevel@tonic-gate /*
5420Sstevel@tonic-gate  * Platform specific page routines
5430Sstevel@tonic-gate  */
5440Sstevel@tonic-gate extern void mach_page_add(page_t **, page_t *);
5450Sstevel@tonic-gate extern void mach_page_sub(page_t **, page_t *);
5460Sstevel@tonic-gate extern uint_t page_get_pagecolors(uint_t);
5470Sstevel@tonic-gate extern void ppcopy_kernel__relocatable(page_t *, page_t *);
5480Sstevel@tonic-gate #define	ppcopy_kernel(p1, p2)	ppcopy_kernel__relocatable(p1, p2)
5490Sstevel@tonic-gate 
5500Sstevel@tonic-gate /*
5510Sstevel@tonic-gate  * platform specific large pages for kernel heap support
5520Sstevel@tonic-gate  */
5530Sstevel@tonic-gate extern size_t get_segkmem_lpsize(size_t lpsize);
5540Sstevel@tonic-gate extern size_t mmu_get_kernel_lpsize(size_t lpsize);
5550Sstevel@tonic-gate extern void mmu_init_kernel_pgsz(struct hat *hat);
5560Sstevel@tonic-gate extern void mmu_init_kcontext();
5570Sstevel@tonic-gate extern uint64_t kcontextreg;
5580Sstevel@tonic-gate 
5590Sstevel@tonic-gate #ifdef	__cplusplus
5600Sstevel@tonic-gate }
5610Sstevel@tonic-gate #endif
5620Sstevel@tonic-gate 
5630Sstevel@tonic-gate #endif	/* _VM_DEP_H */
564