10Sstevel@tonic-gate /* 20Sstevel@tonic-gate * CDDL HEADER START 30Sstevel@tonic-gate * 40Sstevel@tonic-gate * The contents of this file are subject to the terms of the 52241Shuah * Common Development and Distribution License (the "License"). 62241Shuah * You may not use this file except in compliance with the License. 70Sstevel@tonic-gate * 80Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 90Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 100Sstevel@tonic-gate * See the License for the specific language governing permissions 110Sstevel@tonic-gate * and limitations under the License. 120Sstevel@tonic-gate * 130Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 140Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 150Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 160Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 170Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 180Sstevel@tonic-gate * 190Sstevel@tonic-gate * CDDL HEADER END 200Sstevel@tonic-gate */ 210Sstevel@tonic-gate /* 22*4528Spaulsan * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 230Sstevel@tonic-gate * Use is subject to license terms. 240Sstevel@tonic-gate */ 250Sstevel@tonic-gate 260Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 270Sstevel@tonic-gate 280Sstevel@tonic-gate #include <sys/types.h> 290Sstevel@tonic-gate #include <sys/systm.h> 300Sstevel@tonic-gate #include <sys/sysmacros.h> 310Sstevel@tonic-gate #include <sys/archsystm.h> 320Sstevel@tonic-gate #include <sys/vmsystm.h> 330Sstevel@tonic-gate #include <sys/machparam.h> 340Sstevel@tonic-gate #include <sys/machsystm.h> 350Sstevel@tonic-gate #include <vm/vm_dep.h> 360Sstevel@tonic-gate #include <vm/hat_sfmmu.h> 370Sstevel@tonic-gate #include <vm/seg_kmem.h> 380Sstevel@tonic-gate #include <sys/cmn_err.h> 390Sstevel@tonic-gate #include <sys/debug.h> 400Sstevel@tonic-gate #include <sys/cpu_module.h> 410Sstevel@tonic-gate #include <sys/sysmacros.h> 420Sstevel@tonic-gate #include <sys/panic.h> 430Sstevel@tonic-gate 440Sstevel@tonic-gate /* 450Sstevel@tonic-gate * pan_disable_ism_large_pages and pan_disable_large_pages are the Panther- 460Sstevel@tonic-gate * specific versions of disable_ism_large_pages and disable_large_pages, 470Sstevel@tonic-gate * and feed back into those two hat variables at hat initialization time, 480Sstevel@tonic-gate * for Panther-only systems. 490Sstevel@tonic-gate * 502991Ssusans * chpjag_disable_large_pages is the Ch/Jaguar-specific version of 512991Ssusans * disable_large_pages. Ditto for pan_disable_large_pages. 522991Ssusans * Note that the Panther and Ch/Jaguar ITLB do not support 32M/256M pages. 530Sstevel@tonic-gate */ 540Sstevel@tonic-gate static int panther_only = 0; 550Sstevel@tonic-gate 562991Ssusans static uint_t pan_disable_large_pages = (1 << TTE256M); 572991Ssusans static uint_t chjag_disable_large_pages = ((1 << TTE32M) | (1 << TTE256M)); 582991Ssusans 592991Ssusans static uint_t mmu_disable_ism_large_pages = ((1 << TTE64K) | 602659Ssusans (1 << TTE512K) | (1 << TTE32M) | (1 << TTE256M)); 612991Ssusans static uint_t mmu_disable_auto_data_large_pages = ((1 << TTE64K) | 620Sstevel@tonic-gate (1 << TTE512K) | (1 << TTE32M) | (1 << TTE256M)); 632991Ssusans static uint_t mmu_disable_auto_text_large_pages = ((1 << TTE64K) | 642368Sjimand (1 << TTE512K) | (1 << TTE32M) | (1 << TTE256M)); 650Sstevel@tonic-gate 660Sstevel@tonic-gate /* 672991Ssusans * The function returns the USIII+(i)-IV+ mmu-specific values for the 680Sstevel@tonic-gate * hat's disable_large_pages and disable_ism_large_pages variables. 690Sstevel@tonic-gate * Currently the hat's disable_large_pages and disable_ism_large_pages 700Sstevel@tonic-gate * already contain the generic sparc 4 page size info, and the return 710Sstevel@tonic-gate * values are or'd with those values. 720Sstevel@tonic-gate */ 732991Ssusans uint_t 740Sstevel@tonic-gate mmu_large_pages_disabled(uint_t flag) 750Sstevel@tonic-gate { 762991Ssusans uint_t pages_disable = 0; 772991Ssusans extern int use_text_pgsz64K; 782991Ssusans extern int use_text_pgsz512K; 790Sstevel@tonic-gate 802991Ssusans if (flag == HAT_LOAD) { 812991Ssusans if (panther_only) { 820Sstevel@tonic-gate pages_disable = pan_disable_large_pages; 832991Ssusans } else { 842991Ssusans pages_disable = chjag_disable_large_pages; 850Sstevel@tonic-gate } 862991Ssusans } else if (flag == HAT_LOAD_SHARE) { 872991Ssusans pages_disable = mmu_disable_ism_large_pages; 882991Ssusans } else if (flag == HAT_AUTO_DATA) { 892991Ssusans pages_disable = mmu_disable_auto_data_large_pages; 902991Ssusans } else if (flag == HAT_AUTO_TEXT) { 912991Ssusans pages_disable = mmu_disable_auto_text_large_pages; 922991Ssusans if (use_text_pgsz512K) { 932991Ssusans pages_disable &= ~(1 << TTE512K); 942991Ssusans } 952991Ssusans if (use_text_pgsz64K) { 962991Ssusans pages_disable &= ~(1 << TTE64K); 970Sstevel@tonic-gate } 980Sstevel@tonic-gate } 990Sstevel@tonic-gate return (pages_disable); 1000Sstevel@tonic-gate } 1010Sstevel@tonic-gate 1020Sstevel@tonic-gate #if defined(CPU_IMP_DUAL_PAGESIZE) 1030Sstevel@tonic-gate /* 1040Sstevel@tonic-gate * If a platform is running with only Ch+ or Jaguar, and then someone DR's 1050Sstevel@tonic-gate * in a Panther board, the Panther mmu will not like it if one of the already 1060Sstevel@tonic-gate * running threads is context switched to the Panther and tries to program 1070Sstevel@tonic-gate * a 512K or 4M page into the T512_1. So make these platforms pay the price 1080Sstevel@tonic-gate * and follow the Panther DTLB restrictions by default. :) 1090Sstevel@tonic-gate * The mmu_init_mmu_page_sizes code below takes care of heterogeneous 1100Sstevel@tonic-gate * platforms that don't support DR, like daktari. 1110Sstevel@tonic-gate * 1120Sstevel@tonic-gate * The effect of these restrictions is to limit the allowable values in 1130Sstevel@tonic-gate * sfmmu_pgsz[0] and sfmmu_pgsz[1], since these hat variables are used in 1142241Shuah * mmu_set_ctx_page_sizes to set up the values in the sfmmu_cext that 1150Sstevel@tonic-gate * are used at context switch time. The value in sfmmu_pgsz[0] is used in 1160Sstevel@tonic-gate * P_pgsz0 and sfmmu_pgsz[1] is used in P_pgsz1, as per Figure F-1-1 1170Sstevel@tonic-gate * IMMU and DMMU Primary Context Register in the Panther Implementation 1180Sstevel@tonic-gate * Supplement and Table 15-21 DMMU Primary Context Register in the 1190Sstevel@tonic-gate * Cheetah+ Delta PRM. 1200Sstevel@tonic-gate */ 1210Sstevel@tonic-gate #ifdef MIXEDCPU_DR_SUPPORTED 1220Sstevel@tonic-gate int panther_dtlb_restrictions = 1; 1230Sstevel@tonic-gate #else 1240Sstevel@tonic-gate int panther_dtlb_restrictions = 0; 1250Sstevel@tonic-gate #endif /* MIXEDCPU_DR_SUPPORTED */ 1260Sstevel@tonic-gate 1270Sstevel@tonic-gate /* 1280Sstevel@tonic-gate * init_mmu_page_sizes is set to one after the bootup time initialization 1290Sstevel@tonic-gate * via mmu_init_mmu_page_sizes, to indicate that mmu_page_sizes has a 1300Sstevel@tonic-gate * valid value. 1310Sstevel@tonic-gate */ 1320Sstevel@tonic-gate int init_mmu_page_sizes = 0; 1330Sstevel@tonic-gate 1340Sstevel@tonic-gate /* 1350Sstevel@tonic-gate * mmu_init_large_pages is called with the desired ism_pagesize parameter, 1360Sstevel@tonic-gate * for Panther-only systems. It may be called from set_platform_defaults, 1370Sstevel@tonic-gate * if some value other than 32M is desired, for Panther-only systems. 1380Sstevel@tonic-gate * mmu_ism_pagesize is the tunable. If it has a bad value, then only warn, 1390Sstevel@tonic-gate * since it would be bad form to panic due 1400Sstevel@tonic-gate * to a user typo. 1410Sstevel@tonic-gate * 1422991Ssusans * The function re-initializes the disable_ism_large_pages and 1430Sstevel@tonic-gate * pan_disable_large_pages variables, which are closely related. 1440Sstevel@tonic-gate * Aka, if 32M is the desired [D]ISM page sizes, then 256M cannot be allowed 1450Sstevel@tonic-gate * for non-ISM large page usage, or DTLB conflict will occur. Please see the 1460Sstevel@tonic-gate * Panther PRM for additional DTLB technical info. 1470Sstevel@tonic-gate */ 1480Sstevel@tonic-gate void 1490Sstevel@tonic-gate mmu_init_large_pages(size_t ism_pagesize) 1500Sstevel@tonic-gate { 1512241Shuah if (cpu_impl_dual_pgsz == 0) { /* disable_dual_pgsz flag */ 1522991Ssusans pan_disable_large_pages = ((1 << TTE32M) | (1 << TTE256M)); 1532991Ssusans mmu_disable_ism_large_pages = ((1 << TTE64K) | 154*4528Spaulsan (1 << TTE512K) | (1 << TTE32M) | (1 << TTE256M)); 1552991Ssusans mmu_disable_auto_data_large_pages = ((1 << TTE64K) | 156*4528Spaulsan (1 << TTE512K) | (1 << TTE32M) | (1 << TTE256M)); 1570Sstevel@tonic-gate return; 1580Sstevel@tonic-gate } 1590Sstevel@tonic-gate 1600Sstevel@tonic-gate switch (ism_pagesize) { 1610Sstevel@tonic-gate case MMU_PAGESIZE4M: 1622991Ssusans pan_disable_large_pages = (1 << TTE256M); 1632991Ssusans mmu_disable_ism_large_pages = ((1 << TTE64K) | 164*4528Spaulsan (1 << TTE512K) | (1 << TTE32M) | (1 << TTE256M)); 1652991Ssusans mmu_disable_auto_data_large_pages = ((1 << TTE64K) | 166*4528Spaulsan (1 << TTE512K) | (1 << TTE32M) | (1 << TTE256M)); 1670Sstevel@tonic-gate break; 1680Sstevel@tonic-gate case MMU_PAGESIZE32M: 1692991Ssusans pan_disable_large_pages = (1 << TTE256M); 1702991Ssusans mmu_disable_ism_large_pages = ((1 << TTE64K) | 171*4528Spaulsan (1 << TTE512K) | (1 << TTE256M)); 1722991Ssusans mmu_disable_auto_data_large_pages = ((1 << TTE64K) | 173*4528Spaulsan (1 << TTE512K) | (1 << TTE4M) | (1 << TTE256M)); 1742991Ssusans adjust_data_maxlpsize(ism_pagesize); 1750Sstevel@tonic-gate break; 1760Sstevel@tonic-gate case MMU_PAGESIZE256M: 1772991Ssusans pan_disable_large_pages = (1 << TTE32M); 1782991Ssusans mmu_disable_ism_large_pages = ((1 << TTE64K) | 179*4528Spaulsan (1 << TTE512K) | (1 << TTE32M)); 1802991Ssusans mmu_disable_auto_data_large_pages = ((1 << TTE64K) | 181*4528Spaulsan (1 << TTE512K) | (1 << TTE4M) | (1 << TTE32M)); 1822991Ssusans adjust_data_maxlpsize(ism_pagesize); 1830Sstevel@tonic-gate break; 1840Sstevel@tonic-gate default: 1850Sstevel@tonic-gate cmn_err(CE_WARN, "Unrecognized mmu_ism_pagesize value 0x%lx", 186*4528Spaulsan ism_pagesize); 1870Sstevel@tonic-gate break; 1880Sstevel@tonic-gate } 1890Sstevel@tonic-gate } 1900Sstevel@tonic-gate 1910Sstevel@tonic-gate /* 1920Sstevel@tonic-gate * Re-initialize mmu_page_sizes and friends, for Panther mmu support. 1930Sstevel@tonic-gate * Called during very early bootup from check_cpus_set(). 1940Sstevel@tonic-gate * Can be called to verify that mmu_page_sizes are set up correctly. 1950Sstevel@tonic-gate * Note that ncpus is not initialized at this point in the bootup sequence. 1960Sstevel@tonic-gate */ 1970Sstevel@tonic-gate int 1980Sstevel@tonic-gate mmu_init_mmu_page_sizes(int cinfo) 1990Sstevel@tonic-gate { 2000Sstevel@tonic-gate int npanther = cinfo; 2010Sstevel@tonic-gate 2020Sstevel@tonic-gate if (!init_mmu_page_sizes) { 2030Sstevel@tonic-gate if (npanther == ncpunode) { 2040Sstevel@tonic-gate mmu_page_sizes = MMU_PAGE_SIZES; 2050Sstevel@tonic-gate mmu_hashcnt = MAX_HASHCNT; 2062659Ssusans mmu_ism_pagesize = DEFAULT_ISM_PAGESIZE; 2070Sstevel@tonic-gate mmu_exported_pagesize_mask = (1 << TTE8K) | 2080Sstevel@tonic-gate (1 << TTE64K) | (1 << TTE512K) | (1 << TTE4M) | 2090Sstevel@tonic-gate (1 << TTE32M) | (1 << TTE256M); 2100Sstevel@tonic-gate panther_dtlb_restrictions = 1; 2110Sstevel@tonic-gate panther_only = 1; 2120Sstevel@tonic-gate } else if (npanther > 0) { 2130Sstevel@tonic-gate panther_dtlb_restrictions = 1; 2140Sstevel@tonic-gate } 2150Sstevel@tonic-gate init_mmu_page_sizes = 1; 2160Sstevel@tonic-gate return (0); 2170Sstevel@tonic-gate } 2180Sstevel@tonic-gate return (1); 2190Sstevel@tonic-gate } 2200Sstevel@tonic-gate 2210Sstevel@tonic-gate 2220Sstevel@tonic-gate /* Cheetah+ and later worst case DTLB parameters */ 2230Sstevel@tonic-gate #ifndef LOCKED_DTLB_ENTRIES 2240Sstevel@tonic-gate #define LOCKED_DTLB_ENTRIES 5 /* 2 user TSBs, 2 nucleus, + OBP */ 2250Sstevel@tonic-gate #endif 2260Sstevel@tonic-gate #define TOTAL_DTLB_ENTRIES 16 2270Sstevel@tonic-gate #define AVAIL_32M_ENTRIES 0 2280Sstevel@tonic-gate #define AVAIL_256M_ENTRIES 0 2290Sstevel@tonic-gate #define AVAIL_DTLB_ENTRIES (TOTAL_DTLB_ENTRIES - LOCKED_DTLB_ENTRIES) 2300Sstevel@tonic-gate static uint64_t ttecnt_threshold[MMU_PAGE_SIZES] = { 2310Sstevel@tonic-gate AVAIL_DTLB_ENTRIES, AVAIL_DTLB_ENTRIES, 2320Sstevel@tonic-gate AVAIL_DTLB_ENTRIES, AVAIL_DTLB_ENTRIES, 2330Sstevel@tonic-gate AVAIL_32M_ENTRIES, AVAIL_256M_ENTRIES }; 2340Sstevel@tonic-gate 2350Sstevel@tonic-gate /* 2360Sstevel@tonic-gate * The purpose of this code is to indirectly reorganize the sfmmu_pgsz array 2370Sstevel@tonic-gate * in order to handle the Panther mmu DTLB requirements. Panther only supports 2380Sstevel@tonic-gate * the 32M/256M pages in the T512_1 and not in the T16, so the Panther cpu 2390Sstevel@tonic-gate * can only support one of the two largest page sizes at a time (efficiently). 2400Sstevel@tonic-gate * Panther only supports 512K and 4M pages in the T512_0, and 32M/256M pages 2410Sstevel@tonic-gate * in the T512_1. So check the sfmmu flags and ttecnt before enabling 2420Sstevel@tonic-gate * the T512_1 for 32M or 256M page sizes, and make sure that 512K and 4M 2430Sstevel@tonic-gate * requests go to the T512_0. 2440Sstevel@tonic-gate * 2450Sstevel@tonic-gate * The tmp_pgsz array comes into this routine in sorted order, as it is 2460Sstevel@tonic-gate * sorted from largest to smallest #pages per pagesize in use by the hat code, 2470Sstevel@tonic-gate * and leaves with the Panther mmu DTLB requirements satisfied. Note that 2480Sstevel@tonic-gate * when the array leaves this function it may not contain all of the page 2490Sstevel@tonic-gate * size codes that it had coming into the function. 2500Sstevel@tonic-gate * 2510Sstevel@tonic-gate * Note that for DISM the flag can be set but the ttecnt can be 0, if we 2520Sstevel@tonic-gate * didn't fault any pages in. This allows the t512_1 to be reprogrammed, 2530Sstevel@tonic-gate * because the T16 does not support the two giant page sizes. ouch. 2540Sstevel@tonic-gate */ 2550Sstevel@tonic-gate void 2560Sstevel@tonic-gate mmu_fixup_large_pages(struct hat *hat, uint64_t *ttecnt, uint8_t *tmp_pgsz) 2570Sstevel@tonic-gate { 2580Sstevel@tonic-gate uint_t pgsz0 = tmp_pgsz[0]; 2590Sstevel@tonic-gate uint_t pgsz1 = tmp_pgsz[1]; 2600Sstevel@tonic-gate uint_t spgsz; 2610Sstevel@tonic-gate 2620Sstevel@tonic-gate /* 2630Sstevel@tonic-gate * Don't program 2nd dtlb for kernel and ism hat 2640Sstevel@tonic-gate */ 265*4528Spaulsan ASSERT(hat->sfmmu_ismhat == 0); 2660Sstevel@tonic-gate ASSERT(hat != ksfmmup); 2672241Shuah ASSERT(cpu_impl_dual_pgsz == 1); 2680Sstevel@tonic-gate 269*4528Spaulsan ASSERT(!SFMMU_TTEFLAGS_ISSET(hat, HAT_32M_FLAG) || 270*4528Spaulsan !SFMMU_TTEFLAGS_ISSET(hat, HAT_256M_FLAG)); 271*4528Spaulsan ASSERT(!SFMMU_TTEFLAGS_ISSET(hat, HAT_256M_FLAG) || 272*4528Spaulsan !SFMMU_TTEFLAGS_ISSET(hat, HAT_32M_FLAG)); 273*4528Spaulsan ASSERT(!SFMMU_FLAGS_ISSET(hat, HAT_32M_ISM) || 274*4528Spaulsan !SFMMU_FLAGS_ISSET(hat, HAT_256M_ISM)); 275*4528Spaulsan ASSERT(!SFMMU_FLAGS_ISSET(hat, HAT_256M_ISM) || 276*4528Spaulsan !SFMMU_FLAGS_ISSET(hat, HAT_32M_ISM)); 2770Sstevel@tonic-gate 278*4528Spaulsan if (SFMMU_TTEFLAGS_ISSET(hat, HAT_32M_FLAG) || 279*4528Spaulsan (ttecnt[TTE32M] != 0) || 280*4528Spaulsan SFMMU_FLAGS_ISSET(hat, HAT_32M_ISM)) { 281*4528Spaulsan 2820Sstevel@tonic-gate spgsz = pgsz1; 2830Sstevel@tonic-gate pgsz1 = TTE32M; 2840Sstevel@tonic-gate if (pgsz0 == TTE32M) 2850Sstevel@tonic-gate pgsz0 = spgsz; 286*4528Spaulsan 287*4528Spaulsan } else if (SFMMU_TTEFLAGS_ISSET(hat, HAT_256M_FLAG) || 288*4528Spaulsan (ttecnt[TTE256M] != 0) || 289*4528Spaulsan SFMMU_FLAGS_ISSET(hat, HAT_256M_ISM)) { 290*4528Spaulsan 2910Sstevel@tonic-gate spgsz = pgsz1; 2920Sstevel@tonic-gate pgsz1 = TTE256M; 2930Sstevel@tonic-gate if (pgsz0 == TTE256M) 2940Sstevel@tonic-gate pgsz0 = spgsz; 295*4528Spaulsan 2960Sstevel@tonic-gate } else if ((pgsz1 == TTE512K) || (pgsz1 == TTE4M)) { 2970Sstevel@tonic-gate if ((pgsz0 != TTE512K) && (pgsz0 != TTE4M)) { 2980Sstevel@tonic-gate spgsz = pgsz0; 2990Sstevel@tonic-gate pgsz0 = pgsz1; 3000Sstevel@tonic-gate pgsz1 = spgsz; 3010Sstevel@tonic-gate } else { 3020Sstevel@tonic-gate pgsz1 = page_szc(MMU_PAGESIZE); 3030Sstevel@tonic-gate } 3040Sstevel@tonic-gate } 3050Sstevel@tonic-gate /* 3060Sstevel@tonic-gate * This implements PAGESIZE programming of the T8s 3070Sstevel@tonic-gate * if large TTE counts don't exceed the thresholds. 3080Sstevel@tonic-gate */ 3090Sstevel@tonic-gate if (ttecnt[pgsz0] < ttecnt_threshold[pgsz0]) 3100Sstevel@tonic-gate pgsz0 = page_szc(MMU_PAGESIZE); 3110Sstevel@tonic-gate if (ttecnt[pgsz1] < ttecnt_threshold[pgsz1]) 3120Sstevel@tonic-gate pgsz1 = page_szc(MMU_PAGESIZE); 3130Sstevel@tonic-gate tmp_pgsz[0] = pgsz0; 3140Sstevel@tonic-gate tmp_pgsz[1] = pgsz1; 3150Sstevel@tonic-gate } 3160Sstevel@tonic-gate 3170Sstevel@tonic-gate /* 3180Sstevel@tonic-gate * Function to set up the page size values used to reprogram the DTLBs, 3190Sstevel@tonic-gate * when page sizes used by a process change significantly. 3200Sstevel@tonic-gate */ 3210Sstevel@tonic-gate void 3220Sstevel@tonic-gate mmu_setup_page_sizes(struct hat *hat, uint64_t *ttecnt, uint8_t *tmp_pgsz) 3230Sstevel@tonic-gate { 3240Sstevel@tonic-gate uint_t pgsz0, pgsz1; 3250Sstevel@tonic-gate 3260Sstevel@tonic-gate /* 3270Sstevel@tonic-gate * Don't program 2nd dtlb for kernel and ism hat 3280Sstevel@tonic-gate */ 3290Sstevel@tonic-gate ASSERT(hat->sfmmu_ismhat == NULL); 3300Sstevel@tonic-gate ASSERT(hat != ksfmmup); 3310Sstevel@tonic-gate 3322241Shuah if (cpu_impl_dual_pgsz == 0) /* disable_dual_pgsz flag */ 3330Sstevel@tonic-gate return; 3340Sstevel@tonic-gate 3350Sstevel@tonic-gate /* 3360Sstevel@tonic-gate * hat->sfmmu_pgsz[] is an array whose elements 3370Sstevel@tonic-gate * contain a sorted order of page sizes. Element 3380Sstevel@tonic-gate * 0 is the most commonly used page size, followed 3390Sstevel@tonic-gate * by element 1, and so on. 3400Sstevel@tonic-gate * 3410Sstevel@tonic-gate * ttecnt[] is an array of per-page-size page counts 3420Sstevel@tonic-gate * mapped into the process. 3430Sstevel@tonic-gate * 3440Sstevel@tonic-gate * If the HAT's choice for page sizes is unsuitable, 3450Sstevel@tonic-gate * we can override it here. The new values written 3460Sstevel@tonic-gate * to the array will be handed back to us later to 3470Sstevel@tonic-gate * do the actual programming of the TLB hardware. 3480Sstevel@tonic-gate * 3490Sstevel@tonic-gate * The policy we use for programming the dual T8s on 3500Sstevel@tonic-gate * Cheetah+ and beyond is as follows: 3510Sstevel@tonic-gate * 3520Sstevel@tonic-gate * We have two programmable TLBs, so we look at 3530Sstevel@tonic-gate * the two most common page sizes in the array, which 3540Sstevel@tonic-gate * have already been computed for us by the HAT. 3550Sstevel@tonic-gate * If the TTE count of either of a preferred page size 3560Sstevel@tonic-gate * exceeds the number of unlocked T16 entries, 3570Sstevel@tonic-gate * we reprogram one of the T8s to that page size 3580Sstevel@tonic-gate * to avoid thrashing in the T16. Else we program 3590Sstevel@tonic-gate * that T8 to the base page size. Note that we do 3600Sstevel@tonic-gate * not force either T8 to be the base page size if a 3610Sstevel@tonic-gate * process is using more than two page sizes. Policy 3620Sstevel@tonic-gate * decisions about which page sizes are best to use are 3630Sstevel@tonic-gate * left to the upper layers. 3640Sstevel@tonic-gate * 3650Sstevel@tonic-gate * Note that for Panther, 4M and 512K pages need to be 3660Sstevel@tonic-gate * programmed into T512_0, and 32M and 256M into T512_1, 3670Sstevel@tonic-gate * so we don't want to go through the MIN/MAX code. 3680Sstevel@tonic-gate * For partial-Panther systems, we still want to make sure 3690Sstevel@tonic-gate * that 4M and 512K page sizes NEVER get into the T512_1. 3700Sstevel@tonic-gate * Since the DTLB flags are not set up on a per-cpu basis, 3710Sstevel@tonic-gate * Panther rules must be applied for mixed Panther/Cheetah+/ 3720Sstevel@tonic-gate * Jaguar configurations. 3730Sstevel@tonic-gate */ 3740Sstevel@tonic-gate if (panther_dtlb_restrictions) { 3750Sstevel@tonic-gate if ((tmp_pgsz[1] == TTE512K) || (tmp_pgsz[1] == TTE4M)) { 3760Sstevel@tonic-gate if ((tmp_pgsz[0] != TTE512K) && 3770Sstevel@tonic-gate (tmp_pgsz[0] != TTE4M)) { 3780Sstevel@tonic-gate pgsz1 = tmp_pgsz[0]; 3790Sstevel@tonic-gate pgsz0 = tmp_pgsz[1]; 3800Sstevel@tonic-gate } else { 3810Sstevel@tonic-gate pgsz0 = tmp_pgsz[0]; 3820Sstevel@tonic-gate pgsz1 = page_szc(MMU_PAGESIZE); 3830Sstevel@tonic-gate } 3840Sstevel@tonic-gate } else { 3850Sstevel@tonic-gate pgsz0 = tmp_pgsz[0]; 3860Sstevel@tonic-gate pgsz1 = tmp_pgsz[1]; 3870Sstevel@tonic-gate } 3880Sstevel@tonic-gate } else { 3890Sstevel@tonic-gate pgsz0 = MIN(tmp_pgsz[0], tmp_pgsz[1]); 3900Sstevel@tonic-gate pgsz1 = MAX(tmp_pgsz[0], tmp_pgsz[1]); 3910Sstevel@tonic-gate } 3920Sstevel@tonic-gate 3930Sstevel@tonic-gate /* 3940Sstevel@tonic-gate * This implements PAGESIZE programming of the T8s 3950Sstevel@tonic-gate * if large TTE counts don't exceed the thresholds. 3960Sstevel@tonic-gate */ 3970Sstevel@tonic-gate if (ttecnt[pgsz0] < ttecnt_threshold[pgsz0]) 3980Sstevel@tonic-gate pgsz0 = page_szc(MMU_PAGESIZE); 3990Sstevel@tonic-gate if (ttecnt[pgsz1] < ttecnt_threshold[pgsz1]) 4000Sstevel@tonic-gate pgsz1 = page_szc(MMU_PAGESIZE); 4010Sstevel@tonic-gate tmp_pgsz[0] = pgsz0; 4020Sstevel@tonic-gate tmp_pgsz[1] = pgsz1; 4030Sstevel@tonic-gate } 4040Sstevel@tonic-gate 4050Sstevel@tonic-gate /* 4060Sstevel@tonic-gate * The HAT calls this function when an MMU context is allocated so that we 4070Sstevel@tonic-gate * can reprogram the large TLBs appropriately for the new process using 4080Sstevel@tonic-gate * the context. 4090Sstevel@tonic-gate * 4100Sstevel@tonic-gate * The caller must hold the HAT lock. 4110Sstevel@tonic-gate */ 4120Sstevel@tonic-gate void 4130Sstevel@tonic-gate mmu_set_ctx_page_sizes(struct hat *hat) 4140Sstevel@tonic-gate { 4150Sstevel@tonic-gate uint_t pgsz0, pgsz1; 4160Sstevel@tonic-gate uint_t new_cext; 4170Sstevel@tonic-gate 4180Sstevel@tonic-gate ASSERT(sfmmu_hat_lock_held(hat)); 4190Sstevel@tonic-gate ASSERT(hat != ksfmmup); 4200Sstevel@tonic-gate 4212241Shuah if (cpu_impl_dual_pgsz == 0) /* disable_dual_pgsz flag */ 4220Sstevel@tonic-gate return; 4230Sstevel@tonic-gate 4240Sstevel@tonic-gate /* 4250Sstevel@tonic-gate * If supported, reprogram the TLBs to a larger pagesize. 4260Sstevel@tonic-gate */ 4270Sstevel@tonic-gate pgsz0 = hat->sfmmu_pgsz[0]; 4280Sstevel@tonic-gate pgsz1 = hat->sfmmu_pgsz[1]; 4290Sstevel@tonic-gate ASSERT(pgsz0 < mmu_page_sizes); 4300Sstevel@tonic-gate ASSERT(pgsz1 < mmu_page_sizes); 4310Sstevel@tonic-gate #ifdef DEBUG 4320Sstevel@tonic-gate if (panther_dtlb_restrictions) { 4330Sstevel@tonic-gate ASSERT(pgsz1 != TTE512K); 4340Sstevel@tonic-gate ASSERT(pgsz1 != TTE4M); 4350Sstevel@tonic-gate } 4360Sstevel@tonic-gate if (panther_only) { 4370Sstevel@tonic-gate ASSERT(pgsz0 != TTE32M); 4380Sstevel@tonic-gate ASSERT(pgsz0 != TTE256M); 4390Sstevel@tonic-gate } 4400Sstevel@tonic-gate #endif /* DEBUG */ 4410Sstevel@tonic-gate new_cext = TAGACCEXT_MKSZPAIR(pgsz1, pgsz0); 4420Sstevel@tonic-gate if (hat->sfmmu_cext != new_cext) { 4432241Shuah #ifdef DEBUG 4442241Shuah int i; 4452241Shuah /* 4462241Shuah * assert cnum should be invalid, this is because pagesize 4472241Shuah * can only be changed after a proc's ctxs are invalidated. 4482241Shuah */ 4492241Shuah for (i = 0; i < max_mmu_ctxdoms; i++) { 4502241Shuah ASSERT(hat->sfmmu_ctxs[i].cnum == INVALID_CONTEXT); 4512241Shuah } 4522241Shuah #endif /* DEBUG */ 4530Sstevel@tonic-gate hat->sfmmu_cext = new_cext; 4540Sstevel@tonic-gate } 4552241Shuah 4560Sstevel@tonic-gate /* 4570Sstevel@tonic-gate * sfmmu_setctx_sec() will take care of the 4582241Shuah * rest of the chores reprogramming the hat->sfmmu_cext 4590Sstevel@tonic-gate * page size values into the DTLBs. 4600Sstevel@tonic-gate */ 4610Sstevel@tonic-gate } 4620Sstevel@tonic-gate 4630Sstevel@tonic-gate /* 4640Sstevel@tonic-gate * This function assumes that there are either four or six supported page 4650Sstevel@tonic-gate * sizes and at most two programmable TLBs, so we need to decide which 4660Sstevel@tonic-gate * page sizes are most important and then adjust the TLB page sizes 4670Sstevel@tonic-gate * accordingly (if supported). 4680Sstevel@tonic-gate * 4690Sstevel@tonic-gate * If these assumptions change, this function will need to be 4700Sstevel@tonic-gate * updated to support whatever the new limits are. 4710Sstevel@tonic-gate */ 4720Sstevel@tonic-gate void 4730Sstevel@tonic-gate mmu_check_page_sizes(sfmmu_t *sfmmup, uint64_t *ttecnt) 4740Sstevel@tonic-gate { 4750Sstevel@tonic-gate uint64_t sortcnt[MMU_PAGE_SIZES]; 4760Sstevel@tonic-gate uint8_t tmp_pgsz[MMU_PAGE_SIZES]; 4770Sstevel@tonic-gate uint8_t i, j, max; 4780Sstevel@tonic-gate uint16_t oldval, newval; 4790Sstevel@tonic-gate 4800Sstevel@tonic-gate /* 4810Sstevel@tonic-gate * We only consider reprogramming the TLBs if one or more of 4820Sstevel@tonic-gate * the two most used page sizes changes and we're using 4830Sstevel@tonic-gate * large pages in this process, except for Panther 32M/256M pages, 4840Sstevel@tonic-gate * which the Panther T16 does not support. 4850Sstevel@tonic-gate */ 486*4528Spaulsan if (SFMMU_LGPGS_INUSE(sfmmup)) { 4870Sstevel@tonic-gate /* Sort page sizes. */ 4880Sstevel@tonic-gate for (i = 0; i < mmu_page_sizes; i++) { 4890Sstevel@tonic-gate sortcnt[i] = ttecnt[i]; 4900Sstevel@tonic-gate } 4910Sstevel@tonic-gate for (j = 0; j < mmu_page_sizes; j++) { 4920Sstevel@tonic-gate for (i = mmu_page_sizes - 1, max = 0; i > 0; i--) { 4930Sstevel@tonic-gate if (sortcnt[i] > sortcnt[max]) 4940Sstevel@tonic-gate max = i; 4950Sstevel@tonic-gate } 4960Sstevel@tonic-gate tmp_pgsz[j] = max; 4970Sstevel@tonic-gate sortcnt[max] = 0; 4980Sstevel@tonic-gate } 4990Sstevel@tonic-gate 5000Sstevel@tonic-gate /* 5010Sstevel@tonic-gate * Handle Panther page dtlb calcs separately. The check 5020Sstevel@tonic-gate * for actual or potential 32M/256M pages must occur 5030Sstevel@tonic-gate * every time due to lack of T16 support for them. 5040Sstevel@tonic-gate * The sort works fine for Ch+/Jag, but Panther has 5050Sstevel@tonic-gate * pagesize restrictions for both DTLBs. 5060Sstevel@tonic-gate */ 5070Sstevel@tonic-gate oldval = sfmmup->sfmmu_pgsz[0] << 8 | sfmmup->sfmmu_pgsz[1]; 5080Sstevel@tonic-gate 5090Sstevel@tonic-gate if (panther_only) { 5100Sstevel@tonic-gate mmu_fixup_large_pages(sfmmup, ttecnt, tmp_pgsz); 5110Sstevel@tonic-gate } else { 5120Sstevel@tonic-gate /* Check 2 largest values after the sort. */ 5130Sstevel@tonic-gate mmu_setup_page_sizes(sfmmup, ttecnt, tmp_pgsz); 5140Sstevel@tonic-gate } 5150Sstevel@tonic-gate newval = tmp_pgsz[0] << 8 | tmp_pgsz[1]; 5160Sstevel@tonic-gate if (newval != oldval) { 5172241Shuah sfmmu_reprog_pgsz_arr(sfmmup, tmp_pgsz); 5180Sstevel@tonic-gate } 5190Sstevel@tonic-gate } 5200Sstevel@tonic-gate } 5210Sstevel@tonic-gate 5220Sstevel@tonic-gate #endif /* CPU_IMP_DUAL_PAGESIZE */ 5230Sstevel@tonic-gate 5240Sstevel@tonic-gate struct heap_lp_page_size { 5250Sstevel@tonic-gate int impl; 5260Sstevel@tonic-gate uint_t tte; 5270Sstevel@tonic-gate int use_dt512; 5280Sstevel@tonic-gate }; 5290Sstevel@tonic-gate 5300Sstevel@tonic-gate struct heap_lp_page_size heap_lp_pgsz[] = { 5310Sstevel@tonic-gate 5320Sstevel@tonic-gate {CHEETAH_IMPL, TTE8K, 0}, /* default */ 5330Sstevel@tonic-gate {CHEETAH_IMPL, TTE64K, 0}, 5340Sstevel@tonic-gate {CHEETAH_IMPL, TTE4M, 0}, 5350Sstevel@tonic-gate 5360Sstevel@tonic-gate { CHEETAH_PLUS_IMPL, TTE4M, 1 }, /* default */ 5370Sstevel@tonic-gate { CHEETAH_PLUS_IMPL, TTE4M, 0 }, 5380Sstevel@tonic-gate { CHEETAH_PLUS_IMPL, TTE64K, 1 }, 5390Sstevel@tonic-gate { CHEETAH_PLUS_IMPL, TTE64K, 0 }, 5400Sstevel@tonic-gate { CHEETAH_PLUS_IMPL, TTE8K, 0 }, 5410Sstevel@tonic-gate 5420Sstevel@tonic-gate { JALAPENO_IMPL, TTE4M, 1 }, /* default */ 5430Sstevel@tonic-gate { JALAPENO_IMPL, TTE4M, 0 }, 5440Sstevel@tonic-gate { JALAPENO_IMPL, TTE64K, 1 }, 5450Sstevel@tonic-gate { JALAPENO_IMPL, TTE64K, 0 }, 5460Sstevel@tonic-gate { JALAPENO_IMPL, TTE8K, 0 }, 5470Sstevel@tonic-gate 5480Sstevel@tonic-gate { JAGUAR_IMPL, TTE4M, 1 }, /* default */ 5490Sstevel@tonic-gate { JAGUAR_IMPL, TTE4M, 0 }, 5500Sstevel@tonic-gate { JAGUAR_IMPL, TTE64K, 1 }, 5510Sstevel@tonic-gate { JAGUAR_IMPL, TTE64K, 0 }, 5520Sstevel@tonic-gate { JAGUAR_IMPL, TTE8K, 0 }, 5530Sstevel@tonic-gate 5540Sstevel@tonic-gate { SERRANO_IMPL, TTE4M, 1 }, /* default */ 5550Sstevel@tonic-gate { SERRANO_IMPL, TTE4M, 0 }, 5560Sstevel@tonic-gate { SERRANO_IMPL, TTE64K, 1 }, 5570Sstevel@tonic-gate { SERRANO_IMPL, TTE64K, 0 }, 5580Sstevel@tonic-gate { SERRANO_IMPL, TTE8K, 0 }, 5590Sstevel@tonic-gate 5600Sstevel@tonic-gate { PANTHER_IMPL, TTE4M, 1 }, /* default */ 5610Sstevel@tonic-gate { PANTHER_IMPL, TTE4M, 0 }, 5620Sstevel@tonic-gate { PANTHER_IMPL, TTE64K, 1 }, 5630Sstevel@tonic-gate { PANTHER_IMPL, TTE64K, 0 }, 5640Sstevel@tonic-gate { PANTHER_IMPL, TTE8K, 0 } 5650Sstevel@tonic-gate }; 5660Sstevel@tonic-gate 5670Sstevel@tonic-gate int heaplp_use_dt512 = -1; 5680Sstevel@tonic-gate 5690Sstevel@tonic-gate void 5700Sstevel@tonic-gate mmu_init_kernel_pgsz(struct hat *hat) 5710Sstevel@tonic-gate { 5720Sstevel@tonic-gate uint_t tte = page_szc(segkmem_lpsize); 5730Sstevel@tonic-gate uchar_t new_cext_primary, new_cext_nucleus; 5740Sstevel@tonic-gate 5750Sstevel@tonic-gate if (heaplp_use_dt512 == 0 || tte > TTE4M) { 5760Sstevel@tonic-gate /* do not reprogram dt512 tlb */ 5770Sstevel@tonic-gate tte = TTE8K; 5780Sstevel@tonic-gate } 5790Sstevel@tonic-gate 5800Sstevel@tonic-gate new_cext_nucleus = TAGACCEXT_MKSZPAIR(tte, TTE8K); 5810Sstevel@tonic-gate new_cext_primary = TAGACCEXT_MKSZPAIR(TTE8K, tte); 5820Sstevel@tonic-gate 5830Sstevel@tonic-gate hat->sfmmu_cext = new_cext_primary; 5840Sstevel@tonic-gate kcontextreg = ((uint64_t)new_cext_nucleus << CTXREG_NEXT_SHIFT) | 585*4528Spaulsan ((uint64_t)new_cext_primary << CTXREG_EXT_SHIFT); 5860Sstevel@tonic-gate } 5870Sstevel@tonic-gate 5880Sstevel@tonic-gate size_t 5890Sstevel@tonic-gate mmu_get_kernel_lpsize(size_t lpsize) 5900Sstevel@tonic-gate { 5910Sstevel@tonic-gate struct heap_lp_page_size *p_lpgsz, *pend_lpgsz; 5920Sstevel@tonic-gate int impl = cpunodes[getprocessorid()].implementation; 5930Sstevel@tonic-gate uint_t tte = TTE8K; 5940Sstevel@tonic-gate 5952241Shuah if (cpu_impl_dual_pgsz == 0) { 5962241Shuah heaplp_use_dt512 = 0; 5972241Shuah return (MMU_PAGESIZE); 5982241Shuah } 5992241Shuah 6000Sstevel@tonic-gate pend_lpgsz = (struct heap_lp_page_size *) 6010Sstevel@tonic-gate ((char *)heap_lp_pgsz + sizeof (heap_lp_pgsz)); 6020Sstevel@tonic-gate 6030Sstevel@tonic-gate /* search for a valid segkmem_lpsize */ 6040Sstevel@tonic-gate for (p_lpgsz = heap_lp_pgsz; p_lpgsz < pend_lpgsz; p_lpgsz++) { 6050Sstevel@tonic-gate if (impl != p_lpgsz->impl) 6060Sstevel@tonic-gate continue; 6070Sstevel@tonic-gate 6080Sstevel@tonic-gate if (lpsize == 0) { 6090Sstevel@tonic-gate /* 6100Sstevel@tonic-gate * no setting for segkmem_lpsize in /etc/system 6110Sstevel@tonic-gate * use default from the table 6120Sstevel@tonic-gate */ 6130Sstevel@tonic-gate tte = p_lpgsz->tte; 6140Sstevel@tonic-gate heaplp_use_dt512 = p_lpgsz->use_dt512; 6150Sstevel@tonic-gate break; 6160Sstevel@tonic-gate } 6170Sstevel@tonic-gate 6180Sstevel@tonic-gate if (lpsize == TTEBYTES(p_lpgsz->tte) && 6190Sstevel@tonic-gate (heaplp_use_dt512 == -1 || 620*4528Spaulsan heaplp_use_dt512 == p_lpgsz->use_dt512)) { 6210Sstevel@tonic-gate 6220Sstevel@tonic-gate tte = p_lpgsz->tte; 6230Sstevel@tonic-gate heaplp_use_dt512 = p_lpgsz->use_dt512; 6240Sstevel@tonic-gate 6250Sstevel@tonic-gate /* found a match */ 6260Sstevel@tonic-gate break; 6270Sstevel@tonic-gate } 6280Sstevel@tonic-gate } 6290Sstevel@tonic-gate 6300Sstevel@tonic-gate if (p_lpgsz == pend_lpgsz) { 6310Sstevel@tonic-gate /* nothing found: disable large page kernel heap */ 6320Sstevel@tonic-gate tte = TTE8K; 6330Sstevel@tonic-gate heaplp_use_dt512 = 0; 6340Sstevel@tonic-gate } 6350Sstevel@tonic-gate 6360Sstevel@tonic-gate lpsize = TTEBYTES(tte); 6370Sstevel@tonic-gate 6380Sstevel@tonic-gate return (lpsize); 6390Sstevel@tonic-gate } 640