10Sstevel@tonic-gate/* 20Sstevel@tonic-gate * CDDL HEADER START 30Sstevel@tonic-gate * 40Sstevel@tonic-gate * The contents of this file are subject to the terms of the 56330Sjc25722 * Common Development and Distribution License (the "License"). 66330Sjc25722 * You may not use this file except in compliance with the License. 70Sstevel@tonic-gate * 80Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 90Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 100Sstevel@tonic-gate * See the License for the specific language governing permissions 110Sstevel@tonic-gate * and limitations under the License. 120Sstevel@tonic-gate * 130Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 140Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 150Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 160Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 170Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 180Sstevel@tonic-gate * 190Sstevel@tonic-gate * CDDL HEADER END 200Sstevel@tonic-gate */ 210Sstevel@tonic-gate/* 226330Sjc25722 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 230Sstevel@tonic-gate * Use is subject to license terms. 240Sstevel@tonic-gate * 250Sstevel@tonic-gate * Assembly code support for the Cheetah+ module 260Sstevel@tonic-gate */ 270Sstevel@tonic-gate 280Sstevel@tonic-gate#pragma ident "%Z%%M% %I% %E% SMI" 290Sstevel@tonic-gate 300Sstevel@tonic-gate#if !defined(lint) 310Sstevel@tonic-gate#include "assym.h" 320Sstevel@tonic-gate#endif /* lint */ 330Sstevel@tonic-gate 340Sstevel@tonic-gate#include <sys/asm_linkage.h> 350Sstevel@tonic-gate#include <sys/mmu.h> 360Sstevel@tonic-gate#include <vm/hat_sfmmu.h> 370Sstevel@tonic-gate#include <sys/machparam.h> 380Sstevel@tonic-gate#include <sys/machcpuvar.h> 390Sstevel@tonic-gate#include <sys/machthread.h> 400Sstevel@tonic-gate#include <sys/machtrap.h> 410Sstevel@tonic-gate#include <sys/privregs.h> 420Sstevel@tonic-gate#include <sys/asm_linkage.h> 430Sstevel@tonic-gate#include <sys/trap.h> 440Sstevel@tonic-gate#include <sys/cheetahregs.h> 450Sstevel@tonic-gate#include <sys/us3_module.h> 460Sstevel@tonic-gate#include <sys/xc_impl.h> 470Sstevel@tonic-gate#include <sys/intreg.h> 480Sstevel@tonic-gate#include <sys/async.h> 490Sstevel@tonic-gate#include <sys/clock.h> 500Sstevel@tonic-gate#include <sys/cheetahasm.h> 516330Sjc25722#include <sys/cmpregs.h> 520Sstevel@tonic-gate 530Sstevel@tonic-gate#ifdef TRAPTRACE 540Sstevel@tonic-gate#include <sys/traptrace.h> 550Sstevel@tonic-gate#endif /* TRAPTRACE */ 560Sstevel@tonic-gate 576330Sjc25722 580Sstevel@tonic-gate#if !defined(lint) 590Sstevel@tonic-gate 600Sstevel@tonic-gate/* BEGIN CSTYLED */ 610Sstevel@tonic-gate 620Sstevel@tonic-gate/* 630Sstevel@tonic-gate * Cheetah+ version to reflush an Ecache line by index. 640Sstevel@tonic-gate * 650Sstevel@tonic-gate * By default we assume the Ecache is 2-way so we flush both 660Sstevel@tonic-gate * ways. Even if the cache is direct-mapped no harm will come 670Sstevel@tonic-gate * from performing the flush twice, apart from perhaps a performance 680Sstevel@tonic-gate * penalty. 690Sstevel@tonic-gate * 700Sstevel@tonic-gate * XXX - scr2 not used. 710Sstevel@tonic-gate */ 720Sstevel@tonic-gate#define ECACHE_REFLUSH_LINE(ec_set_size, index, scr2) \ 730Sstevel@tonic-gate ldxa [index]ASI_EC_DIAG, %g0; \ 740Sstevel@tonic-gate ldxa [index + ec_set_size]ASI_EC_DIAG, %g0; 750Sstevel@tonic-gate 760Sstevel@tonic-gate/* 770Sstevel@tonic-gate * Cheetah+ version of ecache_flush_line. Uses Cheetah+ Ecache Displacement 780Sstevel@tonic-gate * Flush feature. 790Sstevel@tonic-gate */ 800Sstevel@tonic-gate#define ECACHE_FLUSH_LINE(physaddr, ec_set_size, scr1, scr2) \ 810Sstevel@tonic-gate sub ec_set_size, 1, scr1; \ 820Sstevel@tonic-gate and physaddr, scr1, scr1; \ 830Sstevel@tonic-gate set CHP_ECACHE_IDX_DISP_FLUSH, scr2; \ 840Sstevel@tonic-gate or scr2, scr1, scr1; \ 850Sstevel@tonic-gate ECACHE_REFLUSH_LINE(ec_set_size, scr1, scr2) 860Sstevel@tonic-gate 870Sstevel@tonic-gate/* END CSTYLED */ 880Sstevel@tonic-gate 890Sstevel@tonic-gate/* 900Sstevel@tonic-gate * Panther version to reflush a line from both the L2 cache and L3 910Sstevel@tonic-gate * cache by the respective indexes. Flushes all ways of the line from 920Sstevel@tonic-gate * each cache. 930Sstevel@tonic-gate * 940Sstevel@tonic-gate * l2_index Index into the L2$ of the line to be flushed. This 950Sstevel@tonic-gate * register will not be modified by this routine. 960Sstevel@tonic-gate * l3_index Index into the L3$ of the line to be flushed. This 970Sstevel@tonic-gate * register will not be modified by this routine. 980Sstevel@tonic-gate * scr2 scratch register. 990Sstevel@tonic-gate * scr3 scratch register. 1000Sstevel@tonic-gate * 1010Sstevel@tonic-gate */ 1020Sstevel@tonic-gate#define PN_ECACHE_REFLUSH_LINE(l2_index, l3_index, scr2, scr3) \ 1030Sstevel@tonic-gate set PN_L2_MAX_SET, scr2; \ 1040Sstevel@tonic-gate set PN_L2_SET_SIZE, scr3; \ 1050Sstevel@tonic-gate1: \ 1060Sstevel@tonic-gate ldxa [l2_index + scr2]ASI_L2_TAG, %g0; \ 1070Sstevel@tonic-gate cmp scr2, %g0; \ 1080Sstevel@tonic-gate bg,a 1b; \ 1090Sstevel@tonic-gate sub scr2, scr3, scr2; \ 1106330Sjc25722 mov 6, scr2; \ 1116330Sjc257227: \ 1126330Sjc25722 cmp scr2, %g0; \ 1136330Sjc25722 bg,a 7b; \ 1146330Sjc25722 sub scr2, 1, scr2; \ 1150Sstevel@tonic-gate set PN_L3_MAX_SET, scr2; \ 1160Sstevel@tonic-gate set PN_L3_SET_SIZE, scr3; \ 1170Sstevel@tonic-gate2: \ 1180Sstevel@tonic-gate ldxa [l3_index + scr2]ASI_EC_DIAG, %g0; \ 1190Sstevel@tonic-gate cmp scr2, %g0; \ 1200Sstevel@tonic-gate bg,a 2b; \ 1210Sstevel@tonic-gate sub scr2, scr3, scr2; 1220Sstevel@tonic-gate 1230Sstevel@tonic-gate/* 1240Sstevel@tonic-gate * Panther version of ecache_flush_line. Flushes the line corresponding 1250Sstevel@tonic-gate * to physaddr from both the L2 cache and the L3 cache. 1260Sstevel@tonic-gate * 1270Sstevel@tonic-gate * physaddr Input: Physical address to flush. 1280Sstevel@tonic-gate * Output: Physical address to flush (preserved). 1290Sstevel@tonic-gate * l2_idx_out Input: scratch register. 1300Sstevel@tonic-gate * Output: Index into the L2$ of the line to be flushed. 1310Sstevel@tonic-gate * l3_idx_out Input: scratch register. 1320Sstevel@tonic-gate * Output: Index into the L3$ of the line to be flushed. 1330Sstevel@tonic-gate * scr3 scratch register. 1340Sstevel@tonic-gate * scr4 scratch register. 1350Sstevel@tonic-gate * 1360Sstevel@tonic-gate */ 1370Sstevel@tonic-gate#define PN_ECACHE_FLUSH_LINE(physaddr, l2_idx_out, l3_idx_out, scr3, scr4) \ 1380Sstevel@tonic-gate set PN_L3_SET_SIZE, l2_idx_out; \ 1390Sstevel@tonic-gate sub l2_idx_out, 1, l2_idx_out; \ 1400Sstevel@tonic-gate and physaddr, l2_idx_out, l3_idx_out; \ 1410Sstevel@tonic-gate set PN_L3_IDX_DISP_FLUSH, l2_idx_out; \ 1420Sstevel@tonic-gate or l2_idx_out, l3_idx_out, l3_idx_out; \ 1430Sstevel@tonic-gate set PN_L2_SET_SIZE, l2_idx_out; \ 1440Sstevel@tonic-gate sub l2_idx_out, 1, l2_idx_out; \ 1450Sstevel@tonic-gate and physaddr, l2_idx_out, l2_idx_out; \ 1460Sstevel@tonic-gate set PN_L2_IDX_DISP_FLUSH, scr3; \ 1470Sstevel@tonic-gate or l2_idx_out, scr3, l2_idx_out; \ 1480Sstevel@tonic-gate PN_ECACHE_REFLUSH_LINE(l2_idx_out, l3_idx_out, scr3, scr4) 1490Sstevel@tonic-gate 1500Sstevel@tonic-gate#endif /* !lint */ 1510Sstevel@tonic-gate 1520Sstevel@tonic-gate/* 1530Sstevel@tonic-gate * Fast ECC error at TL>0 handler 1540Sstevel@tonic-gate * We get here via trap 70 at TL>0->Software trap 0 at TL>0. We enter 1550Sstevel@tonic-gate * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate. 1560Sstevel@tonic-gate * For a complete description of the Fast ECC at TL>0 handling see the 1570Sstevel@tonic-gate * comment block "Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy" in 1580Sstevel@tonic-gate * us3_common_asm.s 1590Sstevel@tonic-gate */ 1600Sstevel@tonic-gate#if defined(lint) 1610Sstevel@tonic-gate 1620Sstevel@tonic-gatevoid 1630Sstevel@tonic-gatefast_ecc_tl1_err(void) 1640Sstevel@tonic-gate{} 1650Sstevel@tonic-gate 1660Sstevel@tonic-gate#else /* lint */ 1670Sstevel@tonic-gate 1680Sstevel@tonic-gate .section ".text" 1690Sstevel@tonic-gate .align 64 1700Sstevel@tonic-gate ENTRY_NP(fast_ecc_tl1_err) 1710Sstevel@tonic-gate 1720Sstevel@tonic-gate /* 1730Sstevel@tonic-gate * This macro turns off the D$/I$ if they are on and saves their 1740Sstevel@tonic-gate * original state in ch_err_tl1_tmp, saves all the %g registers in the 1750Sstevel@tonic-gate * ch_err_tl1_data structure, updates the ch_err_tl1_flags and saves 1760Sstevel@tonic-gate * the %tpc in ch_err_tl1_tpc. At the end of this macro, %g1 will 1770Sstevel@tonic-gate * point to the ch_err_tl1_data structure and the original D$/I$ state 1780Sstevel@tonic-gate * will be saved in ch_err_tl1_tmp. All %g registers except for %g1 1790Sstevel@tonic-gate * will be available. 1800Sstevel@tonic-gate */ 1810Sstevel@tonic-gate CH_ERR_TL1_FECC_ENTER; 1820Sstevel@tonic-gate 1830Sstevel@tonic-gate /* 1840Sstevel@tonic-gate * Get the diagnostic logout data. %g4 must be initialized to 1850Sstevel@tonic-gate * current CEEN state, %g5 must point to logout structure in 1860Sstevel@tonic-gate * ch_err_tl1_data_t. %g3 will contain the nesting count upon 1870Sstevel@tonic-gate * return. 1880Sstevel@tonic-gate */ 1890Sstevel@tonic-gate ldxa [%g0]ASI_ESTATE_ERR, %g4 1900Sstevel@tonic-gate and %g4, EN_REG_CEEN, %g4 1910Sstevel@tonic-gate add %g1, CH_ERR_TL1_LOGOUT, %g5 1920Sstevel@tonic-gate DO_TL1_CPU_LOGOUT(%g3, %g2, %g4, %g5, %g6, %g3, %g4) 1930Sstevel@tonic-gate 1940Sstevel@tonic-gate /* 1950Sstevel@tonic-gate * If the logout nesting count is exceeded, we're probably 1960Sstevel@tonic-gate * not making any progress, try to panic instead. 1970Sstevel@tonic-gate */ 1980Sstevel@tonic-gate cmp %g3, CLO_NESTING_MAX 1990Sstevel@tonic-gate bge fecc_tl1_err 2000Sstevel@tonic-gate nop 2010Sstevel@tonic-gate 2020Sstevel@tonic-gate /* 2030Sstevel@tonic-gate * Save the current CEEN and NCEEN state in %g7 and turn them off 2040Sstevel@tonic-gate * before flushing the Ecache. 2050Sstevel@tonic-gate */ 2060Sstevel@tonic-gate ldxa [%g0]ASI_ESTATE_ERR, %g7 2070Sstevel@tonic-gate andn %g7, EN_REG_CEEN | EN_REG_NCEEN, %g5 2080Sstevel@tonic-gate stxa %g5, [%g0]ASI_ESTATE_ERR 2090Sstevel@tonic-gate membar #Sync 2100Sstevel@tonic-gate 2110Sstevel@tonic-gate /* 2120Sstevel@tonic-gate * Flush the Ecache, using the largest possible cache size with the 2130Sstevel@tonic-gate * smallest possible line size since we can't get the actual sizes 2140Sstevel@tonic-gate * from the cpu_node due to DTLB misses. 2150Sstevel@tonic-gate */ 2160Sstevel@tonic-gate PN_L2_FLUSHALL(%g3, %g4, %g5) 2170Sstevel@tonic-gate 2180Sstevel@tonic-gate set CH_ECACHE_MAX_SIZE, %g4 2190Sstevel@tonic-gate set CH_ECACHE_MIN_LSIZE, %g5 2200Sstevel@tonic-gate 2210Sstevel@tonic-gate GET_CPU_IMPL(%g6) 2220Sstevel@tonic-gate cmp %g6, PANTHER_IMPL 2230Sstevel@tonic-gate bne %xcc, 2f 2240Sstevel@tonic-gate nop 2250Sstevel@tonic-gate set PN_L3_SIZE, %g4 2260Sstevel@tonic-gate2: 2270Sstevel@tonic-gate mov %g6, %g3 2280Sstevel@tonic-gate CHP_ECACHE_FLUSHALL(%g4, %g5, %g3) 2290Sstevel@tonic-gate 2300Sstevel@tonic-gate /* 2310Sstevel@tonic-gate * Restore CEEN and NCEEN to the previous state. 2320Sstevel@tonic-gate */ 2330Sstevel@tonic-gate stxa %g7, [%g0]ASI_ESTATE_ERR 2340Sstevel@tonic-gate membar #Sync 2350Sstevel@tonic-gate 2360Sstevel@tonic-gate /* 2370Sstevel@tonic-gate * If we turned off the D$, then flush it and turn it back on. 2380Sstevel@tonic-gate */ 2390Sstevel@tonic-gate ldxa [%g1 + CH_ERR_TL1_TMP]%asi, %g3 2400Sstevel@tonic-gate andcc %g3, CH_ERR_TSTATE_DC_ON, %g0 2410Sstevel@tonic-gate bz %xcc, 3f 2420Sstevel@tonic-gate nop 2430Sstevel@tonic-gate 2440Sstevel@tonic-gate /* 2450Sstevel@tonic-gate * Flush the D$. 2460Sstevel@tonic-gate */ 2470Sstevel@tonic-gate ASM_LD(%g4, dcache_size) 2480Sstevel@tonic-gate ASM_LD(%g5, dcache_linesize) 2490Sstevel@tonic-gate CH_DCACHE_FLUSHALL(%g4, %g5, %g6) 2500Sstevel@tonic-gate 2510Sstevel@tonic-gate /* 2520Sstevel@tonic-gate * Turn the D$ back on. 2530Sstevel@tonic-gate */ 2540Sstevel@tonic-gate ldxa [%g0]ASI_DCU, %g3 2550Sstevel@tonic-gate or %g3, DCU_DC, %g3 2560Sstevel@tonic-gate stxa %g3, [%g0]ASI_DCU 2570Sstevel@tonic-gate membar #Sync 2580Sstevel@tonic-gate3: 2590Sstevel@tonic-gate /* 2600Sstevel@tonic-gate * If we turned off the I$, then flush it and turn it back on. 2610Sstevel@tonic-gate */ 2620Sstevel@tonic-gate ldxa [%g1 + CH_ERR_TL1_TMP]%asi, %g3 2630Sstevel@tonic-gate andcc %g3, CH_ERR_TSTATE_IC_ON, %g0 2640Sstevel@tonic-gate bz %xcc, 4f 2650Sstevel@tonic-gate nop 2660Sstevel@tonic-gate 2670Sstevel@tonic-gate /* 2680Sstevel@tonic-gate * Flush the I$. Panther has different I$ parameters, and we 2690Sstevel@tonic-gate * can't access the logout I$ params without possibly generating 2700Sstevel@tonic-gate * a MMU miss. 2710Sstevel@tonic-gate */ 2720Sstevel@tonic-gate GET_CPU_IMPL(%g6) 2730Sstevel@tonic-gate set PN_ICACHE_SIZE, %g3 2740Sstevel@tonic-gate set CH_ICACHE_SIZE, %g4 2750Sstevel@tonic-gate mov CH_ICACHE_LSIZE, %g5 2760Sstevel@tonic-gate cmp %g6, PANTHER_IMPL 2770Sstevel@tonic-gate movz %xcc, %g3, %g4 2780Sstevel@tonic-gate movz %xcc, PN_ICACHE_LSIZE, %g5 2790Sstevel@tonic-gate CH_ICACHE_FLUSHALL(%g4, %g5, %g6, %g3) 2800Sstevel@tonic-gate 2810Sstevel@tonic-gate /* 2820Sstevel@tonic-gate * Turn the I$ back on. Changing DCU_IC requires flush. 2830Sstevel@tonic-gate */ 2840Sstevel@tonic-gate ldxa [%g0]ASI_DCU, %g3 2850Sstevel@tonic-gate or %g3, DCU_IC, %g3 2860Sstevel@tonic-gate stxa %g3, [%g0]ASI_DCU 2870Sstevel@tonic-gate flush %g0 2880Sstevel@tonic-gate4: 2890Sstevel@tonic-gate 2900Sstevel@tonic-gate#ifdef TRAPTRACE 2910Sstevel@tonic-gate /* 2920Sstevel@tonic-gate * Get current trap trace entry physical pointer. 2930Sstevel@tonic-gate */ 2940Sstevel@tonic-gate CPU_INDEX(%g6, %g5) 2950Sstevel@tonic-gate sll %g6, TRAPTR_SIZE_SHIFT, %g6 2960Sstevel@tonic-gate set trap_trace_ctl, %g5 2970Sstevel@tonic-gate add %g6, %g5, %g6 2980Sstevel@tonic-gate ld [%g6 + TRAPTR_LIMIT], %g5 2990Sstevel@tonic-gate tst %g5 3000Sstevel@tonic-gate be %icc, skip_traptrace 3010Sstevel@tonic-gate nop 3020Sstevel@tonic-gate ldx [%g6 + TRAPTR_PBASE], %g5 3030Sstevel@tonic-gate ld [%g6 + TRAPTR_OFFSET], %g4 3040Sstevel@tonic-gate add %g5, %g4, %g5 3050Sstevel@tonic-gate 3060Sstevel@tonic-gate /* 3070Sstevel@tonic-gate * Create trap trace entry. 3080Sstevel@tonic-gate */ 3090Sstevel@tonic-gate rd %asi, %g7 3100Sstevel@tonic-gate wr %g0, TRAPTR_ASI, %asi 3110Sstevel@tonic-gate rd STICK, %g4 3120Sstevel@tonic-gate stxa %g4, [%g5 + TRAP_ENT_TICK]%asi 3130Sstevel@tonic-gate rdpr %tl, %g4 3140Sstevel@tonic-gate stha %g4, [%g5 + TRAP_ENT_TL]%asi 3150Sstevel@tonic-gate rdpr %tt, %g4 3160Sstevel@tonic-gate stha %g4, [%g5 + TRAP_ENT_TT]%asi 3170Sstevel@tonic-gate rdpr %tpc, %g4 3180Sstevel@tonic-gate stna %g4, [%g5 + TRAP_ENT_TPC]%asi 3190Sstevel@tonic-gate rdpr %tstate, %g4 3200Sstevel@tonic-gate stxa %g4, [%g5 + TRAP_ENT_TSTATE]%asi 3210Sstevel@tonic-gate stna %sp, [%g5 + TRAP_ENT_SP]%asi 3220Sstevel@tonic-gate stna %g0, [%g5 + TRAP_ENT_TR]%asi 3230Sstevel@tonic-gate wr %g0, %g7, %asi 3240Sstevel@tonic-gate ldxa [%g1 + CH_ERR_TL1_SDW_AFAR]%asi, %g3 3250Sstevel@tonic-gate ldxa [%g1 + CH_ERR_TL1_SDW_AFSR]%asi, %g4 3260Sstevel@tonic-gate wr %g0, TRAPTR_ASI, %asi 3270Sstevel@tonic-gate stna %g3, [%g5 + TRAP_ENT_F1]%asi 3280Sstevel@tonic-gate stna %g4, [%g5 + TRAP_ENT_F2]%asi 3290Sstevel@tonic-gate wr %g0, %g7, %asi 3300Sstevel@tonic-gate ldxa [%g1 + CH_ERR_TL1_AFAR]%asi, %g3 3310Sstevel@tonic-gate ldxa [%g1 + CH_ERR_TL1_AFSR]%asi, %g4 3320Sstevel@tonic-gate wr %g0, TRAPTR_ASI, %asi 3330Sstevel@tonic-gate stna %g3, [%g5 + TRAP_ENT_F3]%asi 3340Sstevel@tonic-gate stna %g4, [%g5 + TRAP_ENT_F4]%asi 3350Sstevel@tonic-gate wr %g0, %g7, %asi 3360Sstevel@tonic-gate 3370Sstevel@tonic-gate /* 3380Sstevel@tonic-gate * Advance trap trace pointer. 3390Sstevel@tonic-gate */ 3400Sstevel@tonic-gate ld [%g6 + TRAPTR_OFFSET], %g5 3410Sstevel@tonic-gate ld [%g6 + TRAPTR_LIMIT], %g4 3420Sstevel@tonic-gate st %g5, [%g6 + TRAPTR_LAST_OFFSET] 3430Sstevel@tonic-gate add %g5, TRAP_ENT_SIZE, %g5 3440Sstevel@tonic-gate sub %g4, TRAP_ENT_SIZE, %g4 3450Sstevel@tonic-gate cmp %g5, %g4 3460Sstevel@tonic-gate movge %icc, 0, %g5 3470Sstevel@tonic-gate st %g5, [%g6 + TRAPTR_OFFSET] 3480Sstevel@tonic-gateskip_traptrace: 3490Sstevel@tonic-gate#endif /* TRAPTRACE */ 3500Sstevel@tonic-gate 3510Sstevel@tonic-gate /* 3520Sstevel@tonic-gate * If nesting count is not zero, skip all the AFSR/AFAR 3530Sstevel@tonic-gate * handling and just do the necessary cache-flushing. 3540Sstevel@tonic-gate */ 3550Sstevel@tonic-gate ldxa [%g1 + CH_ERR_TL1_NEST_CNT]%asi, %g2 3560Sstevel@tonic-gate brnz %g2, 6f 3570Sstevel@tonic-gate nop 3580Sstevel@tonic-gate 3590Sstevel@tonic-gate /* 3600Sstevel@tonic-gate * If a UCU or L3_UCU followed by a WDU has occurred go ahead 3610Sstevel@tonic-gate * and panic since a UE will occur (on the retry) before the 362*7058Skwmc * UCU and WDU messages are enqueued. On a Panther processor, 363*7058Skwmc * we need to also see an L3_WDU before panicking. Note that 364*7058Skwmc * we avoid accessing the _EXT ASIs if not on a Panther. 3650Sstevel@tonic-gate */ 3660Sstevel@tonic-gate ldxa [%g1 + CH_ERR_TL1_SDW_AFSR]%asi, %g3 3670Sstevel@tonic-gate set 1, %g4 3680Sstevel@tonic-gate sllx %g4, C_AFSR_UCU_SHIFT, %g4 3690Sstevel@tonic-gate btst %g4, %g3 ! UCU in original shadow AFSR? 3700Sstevel@tonic-gate bnz %xcc, 5f 371*7058Skwmc nop 372*7058Skwmc GET_CPU_IMPL(%g6) 373*7058Skwmc cmp %g6, PANTHER_IMPL 374*7058Skwmc bne %xcc, 6f ! not Panther, no UCU, skip the rest 375*7058Skwmc nop 3760Sstevel@tonic-gate ldxa [%g1 + CH_ERR_TL1_SDW_AFSR_EXT]%asi, %g3 377*7058Skwmc btst C_AFSR_L3_UCU, %g3 ! L3_UCU in original shadow AFSR_EXT? 378*7058Skwmc bz %xcc, 6f ! neither UCU nor L3_UCU was seen 3790Sstevel@tonic-gate nop 3800Sstevel@tonic-gate5: 3810Sstevel@tonic-gate ldxa [%g1 + CH_ERR_TL1_AFSR]%asi, %g4 ! original AFSR 3820Sstevel@tonic-gate ldxa [%g0]ASI_AFSR, %g3 ! current AFSR 3830Sstevel@tonic-gate or %g3, %g4, %g3 ! %g3 = original + current AFSR 3840Sstevel@tonic-gate set 1, %g4 3850Sstevel@tonic-gate sllx %g4, C_AFSR_WDU_SHIFT, %g4 3860Sstevel@tonic-gate btst %g4, %g3 ! WDU in original or current AFSR? 387*7058Skwmc bz %xcc, 6f ! no WDU, skip remaining tests 388*7058Skwmc nop 389*7058Skwmc GET_CPU_IMPL(%g6) 390*7058Skwmc cmp %g6, PANTHER_IMPL 391*7058Skwmc bne %xcc, fecc_tl1_err ! if not Panther, panic (saw UCU, WDU) 3920Sstevel@tonic-gate nop 393*7058Skwmc ldxa [%g1 + CH_ERR_TL1_SDW_AFSR_EXT]%asi, %g4 ! original AFSR_EXT 394*7058Skwmc set ASI_AFSR_EXT_VA, %g6 ! ASI of current AFSR_EXT 395*7058Skwmc ldxa [%g6]ASI_AFSR, %g3 ! value of current AFSR_EXT 396*7058Skwmc or %g3, %g4, %g3 ! %g3 = original + current AFSR_EXT 397*7058Skwmc btst C_AFSR_L3_WDU, %g3 ! L3_WDU in original or current AFSR? 398*7058Skwmc bnz %xcc, fecc_tl1_err ! panic (saw L3_WDU and UCU or L3_UCU) 399*7058Skwmc nop 4000Sstevel@tonic-gate6: 4010Sstevel@tonic-gate /* 4020Sstevel@tonic-gate * We fall into this macro if we've successfully logged the error in 4030Sstevel@tonic-gate * the ch_err_tl1_data structure and want the PIL15 softint to pick 4040Sstevel@tonic-gate * it up and log it. %g1 must point to the ch_err_tl1_data structure. 4050Sstevel@tonic-gate * Restores the %g registers and issues retry. 4060Sstevel@tonic-gate */ 4070Sstevel@tonic-gate CH_ERR_TL1_EXIT; 4080Sstevel@tonic-gate 4090Sstevel@tonic-gate /* 4100Sstevel@tonic-gate * Establish panic exit label. 4110Sstevel@tonic-gate */ 4120Sstevel@tonic-gate CH_ERR_TL1_PANIC_EXIT(fecc_tl1_err); 4130Sstevel@tonic-gate 4140Sstevel@tonic-gate SET_SIZE(fast_ecc_tl1_err) 4150Sstevel@tonic-gate 4160Sstevel@tonic-gate#endif /* lint */ 4170Sstevel@tonic-gate 4180Sstevel@tonic-gate 4190Sstevel@tonic-gate#if defined(lint) 4200Sstevel@tonic-gate/* 4210Sstevel@tonic-gate * scrubphys - Pass in the aligned physical memory address 4220Sstevel@tonic-gate * that you want to scrub, along with the ecache set size. 4230Sstevel@tonic-gate * 4240Sstevel@tonic-gate * 1) Displacement flush the E$ line corresponding to %addr. 4250Sstevel@tonic-gate * The first ldxa guarantees that the %addr is no longer in 4260Sstevel@tonic-gate * M, O, or E (goes to I or S (if instruction fetch also happens). 4270Sstevel@tonic-gate * 2) "Write" the data using a CAS %addr,%g0,%g0. 4280Sstevel@tonic-gate * The casxa guarantees a transition from I to M or S to M. 4290Sstevel@tonic-gate * 3) Displacement flush the E$ line corresponding to %addr. 4300Sstevel@tonic-gate * The second ldxa pushes the M line out of the ecache, into the 4310Sstevel@tonic-gate * writeback buffers, on the way to memory. 4320Sstevel@tonic-gate * 4) The "membar #Sync" pushes the cache line out of the writeback 4330Sstevel@tonic-gate * buffers onto the bus, on the way to dram finally. 4340Sstevel@tonic-gate * 4350Sstevel@tonic-gate * This is a modified version of the algorithm suggested by Gary Lauterbach. 4360Sstevel@tonic-gate * In theory the CAS %addr,%g0,%g0 is supposed to mark the addr's cache line 4370Sstevel@tonic-gate * as modified, but then we found out that for spitfire, if it misses in the 4380Sstevel@tonic-gate * E$ it will probably install as an M, but if it hits in the E$, then it 4390Sstevel@tonic-gate * will stay E, if the store doesn't happen. So the first displacement flush 4400Sstevel@tonic-gate * should ensure that the CAS will miss in the E$. Arrgh. 4410Sstevel@tonic-gate */ 4420Sstevel@tonic-gate/* ARGSUSED */ 4430Sstevel@tonic-gatevoid 4440Sstevel@tonic-gatescrubphys(uint64_t paddr, int ecache_set_size) 4450Sstevel@tonic-gate{} 4460Sstevel@tonic-gate 4470Sstevel@tonic-gate#else /* lint */ 4480Sstevel@tonic-gate ENTRY(scrubphys) 4490Sstevel@tonic-gate rdpr %pstate, %o4 4500Sstevel@tonic-gate andn %o4, PSTATE_IE | PSTATE_AM, %o5 4510Sstevel@tonic-gate wrpr %o5, %g0, %pstate ! clear IE, AM bits 4520Sstevel@tonic-gate 4530Sstevel@tonic-gate GET_CPU_IMPL(%o5) ! Panther Ecache is flushed differently 4540Sstevel@tonic-gate cmp %o5, PANTHER_IMPL 4550Sstevel@tonic-gate bne scrubphys_1 4560Sstevel@tonic-gate nop 4570Sstevel@tonic-gate PN_ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3, %o5) 4580Sstevel@tonic-gate casxa [%o0]ASI_MEM, %g0, %g0 4590Sstevel@tonic-gate PN_ECACHE_REFLUSH_LINE(%o1, %o2, %o3, %o0) 4600Sstevel@tonic-gate b scrubphys_2 4610Sstevel@tonic-gate nop 4620Sstevel@tonic-gatescrubphys_1: 4630Sstevel@tonic-gate ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3) 4640Sstevel@tonic-gate casxa [%o0]ASI_MEM, %g0, %g0 4650Sstevel@tonic-gate ECACHE_REFLUSH_LINE(%o1, %o2, %o3) 4660Sstevel@tonic-gatescrubphys_2: 4670Sstevel@tonic-gate wrpr %g0, %o4, %pstate ! restore earlier pstate register value 4680Sstevel@tonic-gate 4690Sstevel@tonic-gate retl 4700Sstevel@tonic-gate membar #Sync ! move the data out of the load buffer 4710Sstevel@tonic-gate SET_SIZE(scrubphys) 4720Sstevel@tonic-gate 4730Sstevel@tonic-gate#endif /* lint */ 4740Sstevel@tonic-gate 4750Sstevel@tonic-gate 4760Sstevel@tonic-gate#if defined(lint) 4770Sstevel@tonic-gate/* 4784567Sanbui * clearphys - Pass in the physical memory address of the checkblock 4794567Sanbui * that you want to push out, cleared with a recognizable pattern, 4804567Sanbui * from the ecache. 4814567Sanbui * 4824567Sanbui * To ensure that the ecc gets recalculated after the bad data is cleared, 4834567Sanbui * we must write out enough data to fill the w$ line (64 bytes). So we read 4844567Sanbui * in an entire ecache subblock's worth of data, and write it back out. 4854567Sanbui * Then we overwrite the 16 bytes of bad data with the pattern. 4860Sstevel@tonic-gate */ 4870Sstevel@tonic-gate/* ARGSUSED */ 4880Sstevel@tonic-gatevoid 4890Sstevel@tonic-gateclearphys(uint64_t paddr, int ecache_set_size, int ecache_linesize) 4900Sstevel@tonic-gate{ 4910Sstevel@tonic-gate} 4920Sstevel@tonic-gate 4930Sstevel@tonic-gate#else /* lint */ 4940Sstevel@tonic-gate ENTRY(clearphys) 4950Sstevel@tonic-gate /* turn off IE, AM bits */ 4960Sstevel@tonic-gate rdpr %pstate, %o4 4970Sstevel@tonic-gate andn %o4, PSTATE_IE | PSTATE_AM, %o5 4980Sstevel@tonic-gate wrpr %o5, %g0, %pstate 4990Sstevel@tonic-gate 5000Sstevel@tonic-gate /* turn off NCEEN */ 5010Sstevel@tonic-gate ldxa [%g0]ASI_ESTATE_ERR, %o5 5020Sstevel@tonic-gate andn %o5, EN_REG_NCEEN, %o3 5030Sstevel@tonic-gate stxa %o3, [%g0]ASI_ESTATE_ERR 5040Sstevel@tonic-gate membar #Sync 5050Sstevel@tonic-gate 5064567Sanbui /* align address passed with 64 bytes subblock size */ 5074567Sanbui mov CH_ECACHE_SUBBLK_SIZE, %o2 5084567Sanbui andn %o0, (CH_ECACHE_SUBBLK_SIZE - 1), %g1 5094567Sanbui 5104567Sanbui /* move the good data into the W$ */ 5110Sstevel@tonic-gateclearphys_1: 5120Sstevel@tonic-gate subcc %o2, 8, %o2 5134567Sanbui ldxa [%g1 + %o2]ASI_MEM, %g2 5140Sstevel@tonic-gate bge clearphys_1 5154567Sanbui stxa %g2, [%g1 + %o2]ASI_MEM 5160Sstevel@tonic-gate 5174567Sanbui /* now overwrite the bad data */ 5184567Sanbui setx 0xbadecc00badecc01, %g1, %g2 5194567Sanbui stxa %g2, [%o0]ASI_MEM 5204567Sanbui mov 8, %g1 5214567Sanbui stxa %g2, [%o0 + %g1]ASI_MEM 5224567Sanbui 5230Sstevel@tonic-gate GET_CPU_IMPL(%o3) ! Panther Ecache is flushed differently 5240Sstevel@tonic-gate cmp %o3, PANTHER_IMPL 5250Sstevel@tonic-gate bne clearphys_2 5260Sstevel@tonic-gate nop 5270Sstevel@tonic-gate PN_ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3, %g1) 5280Sstevel@tonic-gate casxa [%o0]ASI_MEM, %g0, %g0 5290Sstevel@tonic-gate PN_ECACHE_REFLUSH_LINE(%o1, %o2, %o3, %o0) 5300Sstevel@tonic-gate b clearphys_3 5310Sstevel@tonic-gate nop 5320Sstevel@tonic-gateclearphys_2: 5330Sstevel@tonic-gate ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3) 5340Sstevel@tonic-gate casxa [%o0]ASI_MEM, %g0, %g0 5350Sstevel@tonic-gate ECACHE_REFLUSH_LINE(%o1, %o2, %o3) 5360Sstevel@tonic-gateclearphys_3: 5370Sstevel@tonic-gate /* clear the AFSR */ 5380Sstevel@tonic-gate ldxa [%g0]ASI_AFSR, %o1 5390Sstevel@tonic-gate stxa %o1, [%g0]ASI_AFSR 5400Sstevel@tonic-gate membar #Sync 5410Sstevel@tonic-gate 5420Sstevel@tonic-gate /* turn NCEEN back on */ 5430Sstevel@tonic-gate stxa %o5, [%g0]ASI_ESTATE_ERR 5440Sstevel@tonic-gate membar #Sync 5450Sstevel@tonic-gate 5460Sstevel@tonic-gate /* return and re-enable IE and AM */ 5470Sstevel@tonic-gate retl 5480Sstevel@tonic-gate wrpr %g0, %o4, %pstate 5490Sstevel@tonic-gate SET_SIZE(clearphys) 5500Sstevel@tonic-gate 5510Sstevel@tonic-gate#endif /* lint */ 5520Sstevel@tonic-gate 5530Sstevel@tonic-gate 5540Sstevel@tonic-gate#if defined(lint) 5550Sstevel@tonic-gate/* 5560Sstevel@tonic-gate * Cheetah+ Ecache displacement flush the specified line from the E$ 5570Sstevel@tonic-gate * 5580Sstevel@tonic-gate * For Panther, this means flushing the specified line from both the 5590Sstevel@tonic-gate * L2 cache and L3 cache. 5600Sstevel@tonic-gate * 5610Sstevel@tonic-gate * Register usage: 5620Sstevel@tonic-gate * %o0 - 64 bit physical address for flushing 5630Sstevel@tonic-gate * %o1 - Ecache set size 5640Sstevel@tonic-gate */ 5650Sstevel@tonic-gate/*ARGSUSED*/ 5660Sstevel@tonic-gatevoid 5670Sstevel@tonic-gateecache_flush_line(uint64_t flushaddr, int ec_set_size) 5680Sstevel@tonic-gate{ 5690Sstevel@tonic-gate} 5700Sstevel@tonic-gate#else /* lint */ 5710Sstevel@tonic-gate ENTRY(ecache_flush_line) 5720Sstevel@tonic-gate 5730Sstevel@tonic-gate GET_CPU_IMPL(%o3) ! Panther Ecache is flushed differently 5740Sstevel@tonic-gate cmp %o3, PANTHER_IMPL 5750Sstevel@tonic-gate bne ecache_flush_line_1 5760Sstevel@tonic-gate nop 5770Sstevel@tonic-gate 5780Sstevel@tonic-gate PN_ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3, %o4) 5790Sstevel@tonic-gate b ecache_flush_line_2 5800Sstevel@tonic-gate nop 5810Sstevel@tonic-gateecache_flush_line_1: 5820Sstevel@tonic-gate ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3) 5830Sstevel@tonic-gateecache_flush_line_2: 5840Sstevel@tonic-gate retl 5850Sstevel@tonic-gate nop 5860Sstevel@tonic-gate SET_SIZE(ecache_flush_line) 5870Sstevel@tonic-gate#endif /* lint */ 5880Sstevel@tonic-gate 5890Sstevel@tonic-gate#if defined(lint) 5900Sstevel@tonic-gatevoid 5910Sstevel@tonic-gateset_afsr_ext(uint64_t afsr_ext) 5920Sstevel@tonic-gate{ 5930Sstevel@tonic-gate afsr_ext = afsr_ext; 5940Sstevel@tonic-gate} 5950Sstevel@tonic-gate#else /* lint */ 5960Sstevel@tonic-gate 5970Sstevel@tonic-gate ENTRY(set_afsr_ext) 5980Sstevel@tonic-gate set ASI_AFSR_EXT_VA, %o1 5990Sstevel@tonic-gate stxa %o0, [%o1]ASI_AFSR ! afsr_ext reg 6000Sstevel@tonic-gate membar #Sync 6010Sstevel@tonic-gate retl 6020Sstevel@tonic-gate nop 6030Sstevel@tonic-gate SET_SIZE(set_afsr_ext) 6040Sstevel@tonic-gate 6050Sstevel@tonic-gate#endif /* lint */ 6060Sstevel@tonic-gate 6070Sstevel@tonic-gate 6080Sstevel@tonic-gate#if defined(lint) 6090Sstevel@tonic-gate/* 6100Sstevel@tonic-gate * The CPU jumps here from the MMU exception handler if an ITLB parity 6110Sstevel@tonic-gate * error is detected and we are running on Panther. 6120Sstevel@tonic-gate * 6130Sstevel@tonic-gate * In this routine we collect diagnostic information and write it to our 6140Sstevel@tonic-gate * logout structure (if possible) and clear all ITLB entries that may have 6150Sstevel@tonic-gate * caused our parity trap. 6160Sstevel@tonic-gate * Then we call cpu_tlb_parity_error via systrap in order to drop down to TL0 6170Sstevel@tonic-gate * and log any error messages. As for parameters to cpu_tlb_parity_error, we 6180Sstevel@tonic-gate * send two: 6190Sstevel@tonic-gate * 6200Sstevel@tonic-gate * %g2 - Contains the VA whose lookup in the ITLB caused the parity error 6210Sstevel@tonic-gate * %g3 - Contains the tlo_info field of the pn_tlb_logout logout struct, 6220Sstevel@tonic-gate * regardless of whether or not we actually used the logout struct. 6230Sstevel@tonic-gate * 6240Sstevel@tonic-gate * In the TL0 handler (cpu_tlb_parity_error) we will compare those two 6250Sstevel@tonic-gate * parameters to the data contained in the logout structure in order to 6260Sstevel@tonic-gate * determine whether the logout information is valid for this particular 6270Sstevel@tonic-gate * error or not. 6280Sstevel@tonic-gate */ 6290Sstevel@tonic-gatevoid 6300Sstevel@tonic-gateitlb_parity_trap(void) 6310Sstevel@tonic-gate{} 6320Sstevel@tonic-gate 6330Sstevel@tonic-gate#else /* lint */ 6340Sstevel@tonic-gate 6350Sstevel@tonic-gate ENTRY_NP(itlb_parity_trap) 6360Sstevel@tonic-gate /* 6370Sstevel@tonic-gate * Collect important information about the trap which will be 6380Sstevel@tonic-gate * used as a parameter to the TL0 handler. 6390Sstevel@tonic-gate */ 6400Sstevel@tonic-gate wr %g0, ASI_IMMU, %asi 6410Sstevel@tonic-gate rdpr %tpc, %g2 ! VA that caused the IMMU trap 6420Sstevel@tonic-gate ldxa [MMU_TAG_ACCESS_EXT]%asi, %g3 ! read the trap VA page size 6430Sstevel@tonic-gate set PN_ITLB_PGSZ_MASK, %g4 6440Sstevel@tonic-gate and %g3, %g4, %g3 6450Sstevel@tonic-gate ldxa [MMU_TAG_ACCESS]%asi, %g4 6460Sstevel@tonic-gate set TAGREAD_CTX_MASK, %g5 6470Sstevel@tonic-gate and %g4, %g5, %g4 6480Sstevel@tonic-gate or %g4, %g3, %g3 ! 'or' in the trap context and 6490Sstevel@tonic-gate mov 1, %g4 ! add the IMMU flag to complete 6500Sstevel@tonic-gate sllx %g4, PN_TLO_INFO_IMMU_SHIFT, %g4 6510Sstevel@tonic-gate or %g4, %g3, %g3 ! the tlo_info field for logout 6520Sstevel@tonic-gate stxa %g0,[MMU_SFSR]%asi ! clear the SFSR 6530Sstevel@tonic-gate membar #Sync 6540Sstevel@tonic-gate 6550Sstevel@tonic-gate /* 6560Sstevel@tonic-gate * at this point: 6570Sstevel@tonic-gate * %g2 - contains the VA whose lookup caused the trap 6580Sstevel@tonic-gate * %g3 - contains the tlo_info field 6590Sstevel@tonic-gate * 6600Sstevel@tonic-gate * Next, we calculate the TLB index value for the failing VA. 6610Sstevel@tonic-gate */ 6620Sstevel@tonic-gate mov %g2, %g4 ! We need the ITLB index 6630Sstevel@tonic-gate set PN_ITLB_PGSZ_MASK, %g5 6640Sstevel@tonic-gate and %g3, %g5, %g5 6650Sstevel@tonic-gate srlx %g5, PN_ITLB_PGSZ_SHIFT, %g5 6660Sstevel@tonic-gate PN_GET_TLB_INDEX(%g4, %g5) ! %g4 has the index 6670Sstevel@tonic-gate sllx %g4, PN_TLB_ACC_IDX_SHIFT, %g4 ! shift the index into place 6680Sstevel@tonic-gate set PN_ITLB_T512, %g5 6690Sstevel@tonic-gate or %g4, %g5, %g4 ! and add in the TLB ID 6700Sstevel@tonic-gate 6710Sstevel@tonic-gate /* 6720Sstevel@tonic-gate * at this point: 6730Sstevel@tonic-gate * %g2 - contains the VA whose lookup caused the trap 6740Sstevel@tonic-gate * %g3 - contains the tlo_info field 6750Sstevel@tonic-gate * %g4 - contains the TLB access index value for the 6760Sstevel@tonic-gate * VA/PgSz in question 6770Sstevel@tonic-gate * 6780Sstevel@tonic-gate * Check to see if the logout structure is available. 6790Sstevel@tonic-gate */ 6800Sstevel@tonic-gate set CHPR_TLB_LOGOUT, %g6 6810Sstevel@tonic-gate GET_CPU_PRIVATE_PTR(%g6, %g1, %g5, itlb_parity_trap_1) 6820Sstevel@tonic-gate set LOGOUT_INVALID_U32, %g6 6830Sstevel@tonic-gate sllx %g6, 32, %g6 ! if our logout structure is 6840Sstevel@tonic-gate set LOGOUT_INVALID_L32, %g5 ! unavailable or if it is 6850Sstevel@tonic-gate or %g5, %g6, %g5 ! already being used, then we 6860Sstevel@tonic-gate ldx [%g1 + PN_TLO_ADDR], %g6 ! don't collect any diagnostic 6870Sstevel@tonic-gate cmp %g6, %g5 ! information before clearing 6880Sstevel@tonic-gate bne itlb_parity_trap_1 ! and logging the error. 6890Sstevel@tonic-gate nop 6900Sstevel@tonic-gate 6910Sstevel@tonic-gate /* 6920Sstevel@tonic-gate * Record the logout information. %g4 contains our index + TLB ID 6930Sstevel@tonic-gate * for use in ASI_ITLB_ACCESS and ASI_ITLB_TAGREAD. %g1 contains 6940Sstevel@tonic-gate * the pointer to our logout struct. 6950Sstevel@tonic-gate */ 6960Sstevel@tonic-gate stx %g3, [%g1 + PN_TLO_INFO] 6970Sstevel@tonic-gate stx %g2, [%g1 + PN_TLO_ADDR] 6980Sstevel@tonic-gate stx %g2, [%g1 + PN_TLO_PC] ! %tpc == fault addr for IMMU 6990Sstevel@tonic-gate 7000Sstevel@tonic-gate add %g1, PN_TLO_ITLB_TTE, %g1 ! move up the pointer 7010Sstevel@tonic-gate 7020Sstevel@tonic-gate ldxa [%g4]ASI_ITLB_ACCESS, %g5 ! read the data 7030Sstevel@tonic-gate stx %g5, [%g1 + CH_TLO_TTE_DATA] ! store it away 7040Sstevel@tonic-gate ldxa [%g4]ASI_ITLB_TAGREAD, %g5 ! read the tag 7050Sstevel@tonic-gate stx %g5, [%g1 + CH_TLO_TTE_TAG] ! store it away 7060Sstevel@tonic-gate 7070Sstevel@tonic-gate set PN_TLB_ACC_WAY_BIT, %g6 ! same thing again for way 1 7080Sstevel@tonic-gate or %g4, %g6, %g4 7090Sstevel@tonic-gate add %g1, CH_TLO_TTE_SIZE, %g1 ! move up the pointer 7100Sstevel@tonic-gate 7110Sstevel@tonic-gate ldxa [%g4]ASI_ITLB_ACCESS, %g5 ! read the data 7120Sstevel@tonic-gate stx %g5, [%g1 + CH_TLO_TTE_DATA] ! store it away 7130Sstevel@tonic-gate ldxa [%g4]ASI_ITLB_TAGREAD, %g5 ! read the tag 7140Sstevel@tonic-gate stx %g5, [%g1 + CH_TLO_TTE_TAG] ! store it away 7150Sstevel@tonic-gate 7160Sstevel@tonic-gate andn %g4, %g6, %g4 ! back to way 0 7170Sstevel@tonic-gate 7180Sstevel@tonic-gateitlb_parity_trap_1: 7190Sstevel@tonic-gate /* 7200Sstevel@tonic-gate * at this point: 7210Sstevel@tonic-gate * %g2 - contains the VA whose lookup caused the trap 7220Sstevel@tonic-gate * %g3 - contains the tlo_info field 7230Sstevel@tonic-gate * %g4 - contains the TLB access index value for the 7240Sstevel@tonic-gate * VA/PgSz in question 7250Sstevel@tonic-gate * 7260Sstevel@tonic-gate * Here we will clear the errors from the TLB. 7270Sstevel@tonic-gate */ 7280Sstevel@tonic-gate set MMU_TAG_ACCESS, %g5 ! We write a TTE tag value of 7290Sstevel@tonic-gate stxa %g0, [%g5]ASI_IMMU ! 0 as it will be invalid. 7300Sstevel@tonic-gate stxa %g0, [%g4]ASI_ITLB_ACCESS ! Write the data and tag 7310Sstevel@tonic-gate membar #Sync 7320Sstevel@tonic-gate 7330Sstevel@tonic-gate set PN_TLB_ACC_WAY_BIT, %g6 ! same thing again for way 1 7340Sstevel@tonic-gate or %g4, %g6, %g4 7350Sstevel@tonic-gate 7360Sstevel@tonic-gate stxa %g0, [%g4]ASI_ITLB_ACCESS ! Write same data and tag 7370Sstevel@tonic-gate membar #Sync 7380Sstevel@tonic-gate 7390Sstevel@tonic-gate sethi %hi(FLUSH_ADDR), %g6 ! PRM says we need to issue a 7400Sstevel@tonic-gate flush %g6 ! flush after writing MMU regs 7410Sstevel@tonic-gate 7420Sstevel@tonic-gate /* 7430Sstevel@tonic-gate * at this point: 7440Sstevel@tonic-gate * %g2 - contains the VA whose lookup caused the trap 7450Sstevel@tonic-gate * %g3 - contains the tlo_info field 7460Sstevel@tonic-gate * 7470Sstevel@tonic-gate * Call cpu_tlb_parity_error via systrap at PIL 14 unless we're 7480Sstevel@tonic-gate * already at PIL 15. */ 7490Sstevel@tonic-gate set cpu_tlb_parity_error, %g1 7500Sstevel@tonic-gate rdpr %pil, %g4 7510Sstevel@tonic-gate cmp %g4, PIL_14 7520Sstevel@tonic-gate movl %icc, PIL_14, %g4 7530Sstevel@tonic-gate ba sys_trap 7540Sstevel@tonic-gate nop 7550Sstevel@tonic-gate SET_SIZE(itlb_parity_trap) 7560Sstevel@tonic-gate 7570Sstevel@tonic-gate#endif /* lint */ 7580Sstevel@tonic-gate 7590Sstevel@tonic-gate#if defined(lint) 7600Sstevel@tonic-gate/* 7610Sstevel@tonic-gate * The CPU jumps here from the MMU exception handler if a DTLB parity 7620Sstevel@tonic-gate * error is detected and we are running on Panther. 7630Sstevel@tonic-gate * 7640Sstevel@tonic-gate * In this routine we collect diagnostic information and write it to our 7650Sstevel@tonic-gate * logout structure (if possible) and clear all DTLB entries that may have 7660Sstevel@tonic-gate * caused our parity trap. 7670Sstevel@tonic-gate * Then we call cpu_tlb_parity_error via systrap in order to drop down to TL0 7680Sstevel@tonic-gate * and log any error messages. As for parameters to cpu_tlb_parity_error, we 7690Sstevel@tonic-gate * send two: 7700Sstevel@tonic-gate * 7710Sstevel@tonic-gate * %g2 - Contains the VA whose lookup in the DTLB caused the parity error 7720Sstevel@tonic-gate * %g3 - Contains the tlo_info field of the pn_tlb_logout logout struct, 7730Sstevel@tonic-gate * regardless of whether or not we actually used the logout struct. 7740Sstevel@tonic-gate * 7750Sstevel@tonic-gate * In the TL0 handler (cpu_tlb_parity_error) we will compare those two 7760Sstevel@tonic-gate * parameters to the data contained in the logout structure in order to 7770Sstevel@tonic-gate * determine whether the logout information is valid for this particular 7780Sstevel@tonic-gate * error or not. 7790Sstevel@tonic-gate */ 7800Sstevel@tonic-gatevoid 7810Sstevel@tonic-gatedtlb_parity_trap(void) 7820Sstevel@tonic-gate{} 7830Sstevel@tonic-gate 7840Sstevel@tonic-gate#else /* lint */ 7850Sstevel@tonic-gate 7860Sstevel@tonic-gate ENTRY_NP(dtlb_parity_trap) 7870Sstevel@tonic-gate /* 7880Sstevel@tonic-gate * Collect important information about the trap which will be 7890Sstevel@tonic-gate * used as a parameter to the TL0 handler. 7900Sstevel@tonic-gate */ 7910Sstevel@tonic-gate wr %g0, ASI_DMMU, %asi 7920Sstevel@tonic-gate ldxa [MMU_SFAR]%asi, %g2 ! VA that caused the IMMU trap 7930Sstevel@tonic-gate ldxa [MMU_TAG_ACCESS_EXT]%asi, %g3 ! read the trap VA page sizes 7940Sstevel@tonic-gate set PN_DTLB_PGSZ_MASK, %g4 7950Sstevel@tonic-gate and %g3, %g4, %g3 7960Sstevel@tonic-gate ldxa [MMU_TAG_ACCESS]%asi, %g4 7970Sstevel@tonic-gate set TAGREAD_CTX_MASK, %g5 ! 'or' in the trap context 7980Sstevel@tonic-gate and %g4, %g5, %g4 ! to complete the tlo_info 7990Sstevel@tonic-gate or %g4, %g3, %g3 ! field for logout 8000Sstevel@tonic-gate stxa %g0,[MMU_SFSR]%asi ! clear the SFSR 8010Sstevel@tonic-gate membar #Sync 8020Sstevel@tonic-gate 8030Sstevel@tonic-gate /* 8040Sstevel@tonic-gate * at this point: 8050Sstevel@tonic-gate * %g2 - contains the VA whose lookup caused the trap 8060Sstevel@tonic-gate * %g3 - contains the tlo_info field 8070Sstevel@tonic-gate * 8080Sstevel@tonic-gate * Calculate the TLB index values for the failing VA. Since the T512 8090Sstevel@tonic-gate * TLBs can be configured for different page sizes, we need to find 8100Sstevel@tonic-gate * the index into each one separately. 8110Sstevel@tonic-gate */ 8120Sstevel@tonic-gate mov %g2, %g4 ! First we get the DTLB_0 index 8130Sstevel@tonic-gate set PN_DTLB_PGSZ0_MASK, %g5 8140Sstevel@tonic-gate and %g3, %g5, %g5 8150Sstevel@tonic-gate srlx %g5, PN_DTLB_PGSZ0_SHIFT, %g5 8160Sstevel@tonic-gate PN_GET_TLB_INDEX(%g4, %g5) ! %g4 has the DTLB_0 index 8170Sstevel@tonic-gate sllx %g4, PN_TLB_ACC_IDX_SHIFT, %g4 ! shift the index into place 8180Sstevel@tonic-gate set PN_DTLB_T512_0, %g5 8190Sstevel@tonic-gate or %g4, %g5, %g4 ! and add in the TLB ID 8200Sstevel@tonic-gate 8210Sstevel@tonic-gate mov %g2, %g7 ! Next we get the DTLB_1 index 8220Sstevel@tonic-gate set PN_DTLB_PGSZ1_MASK, %g5 8230Sstevel@tonic-gate and %g3, %g5, %g5 8240Sstevel@tonic-gate srlx %g5, PN_DTLB_PGSZ1_SHIFT, %g5 8250Sstevel@tonic-gate PN_GET_TLB_INDEX(%g7, %g5) ! %g7 has the DTLB_1 index 8260Sstevel@tonic-gate sllx %g7, PN_TLB_ACC_IDX_SHIFT, %g7 ! shift the index into place 8270Sstevel@tonic-gate set PN_DTLB_T512_1, %g5 8280Sstevel@tonic-gate or %g7, %g5, %g7 ! and add in the TLB ID 8290Sstevel@tonic-gate 8300Sstevel@tonic-gate /* 8310Sstevel@tonic-gate * at this point: 8320Sstevel@tonic-gate * %g2 - contains the VA whose lookup caused the trap 8330Sstevel@tonic-gate * %g3 - contains the tlo_info field 8340Sstevel@tonic-gate * %g4 - contains the T512_0 access index value for the 8350Sstevel@tonic-gate * VA/PgSz in question 8360Sstevel@tonic-gate * %g7 - contains the T512_1 access index value for the 8370Sstevel@tonic-gate * VA/PgSz in question 8380Sstevel@tonic-gate * 8390Sstevel@tonic-gate * If this trap happened at TL>0, then we don't want to mess 8400Sstevel@tonic-gate * with the normal logout struct since that could caused a TLB 8410Sstevel@tonic-gate * miss. 8420Sstevel@tonic-gate */ 8430Sstevel@tonic-gate rdpr %tl, %g6 ! read current trap level 8440Sstevel@tonic-gate cmp %g6, 1 ! skip over the tl>1 code 8450Sstevel@tonic-gate ble dtlb_parity_trap_1 ! if TL <= 1. 8460Sstevel@tonic-gate nop 8470Sstevel@tonic-gate 8480Sstevel@tonic-gate /* 8490Sstevel@tonic-gate * If we are here, then the trap happened at TL>1. Simply 8500Sstevel@tonic-gate * update our tlo_info field and then skip to the TLB flush 8510Sstevel@tonic-gate * code. 8520Sstevel@tonic-gate */ 8530Sstevel@tonic-gate mov 1, %g6 8540Sstevel@tonic-gate sllx %g6, PN_TLO_INFO_TL1_SHIFT, %g6 8550Sstevel@tonic-gate or %g6, %g3, %g3 8560Sstevel@tonic-gate ba dtlb_parity_trap_2 8570Sstevel@tonic-gate nop 8580Sstevel@tonic-gate 8590Sstevel@tonic-gatedtlb_parity_trap_1: 8600Sstevel@tonic-gate /* 8610Sstevel@tonic-gate * at this point: 8620Sstevel@tonic-gate * %g2 - contains the VA whose lookup caused the trap 8630Sstevel@tonic-gate * %g3 - contains the tlo_info field 8640Sstevel@tonic-gate * %g4 - contains the T512_0 access index value for the 8650Sstevel@tonic-gate * VA/PgSz in question 8660Sstevel@tonic-gate * %g7 - contains the T512_1 access index value for the 8670Sstevel@tonic-gate * VA/PgSz in question 8680Sstevel@tonic-gate * 8690Sstevel@tonic-gate * Check to see if the logout structure is available. 8700Sstevel@tonic-gate */ 8710Sstevel@tonic-gate set CHPR_TLB_LOGOUT, %g6 8720Sstevel@tonic-gate GET_CPU_PRIVATE_PTR(%g6, %g1, %g5, dtlb_parity_trap_2) 8730Sstevel@tonic-gate set LOGOUT_INVALID_U32, %g6 8740Sstevel@tonic-gate sllx %g6, 32, %g6 ! if our logout structure is 8750Sstevel@tonic-gate set LOGOUT_INVALID_L32, %g5 ! unavailable or if it is 8760Sstevel@tonic-gate or %g5, %g6, %g5 ! already being used, then we 8770Sstevel@tonic-gate ldx [%g1 + PN_TLO_ADDR], %g6 ! don't collect any diagnostic 8780Sstevel@tonic-gate cmp %g6, %g5 ! information before clearing 8790Sstevel@tonic-gate bne dtlb_parity_trap_2 ! and logging the error. 8800Sstevel@tonic-gate nop 8810Sstevel@tonic-gate 8820Sstevel@tonic-gate /* 8830Sstevel@tonic-gate * Record the logout information. %g4 contains our DTLB_0 8840Sstevel@tonic-gate * index + TLB ID and %g7 contains our DTLB_1 index + TLB ID 8850Sstevel@tonic-gate * both of which will be used for ASI_DTLB_ACCESS and 8860Sstevel@tonic-gate * ASI_DTLB_TAGREAD. %g1 contains the pointer to our logout 8870Sstevel@tonic-gate * struct. 8880Sstevel@tonic-gate */ 8890Sstevel@tonic-gate stx %g3, [%g1 + PN_TLO_INFO] 8900Sstevel@tonic-gate stx %g2, [%g1 + PN_TLO_ADDR] 8910Sstevel@tonic-gate rdpr %tpc, %g5 8920Sstevel@tonic-gate stx %g5, [%g1 + PN_TLO_PC] 8930Sstevel@tonic-gate 8940Sstevel@tonic-gate add %g1, PN_TLO_DTLB_TTE, %g1 ! move up the pointer 8950Sstevel@tonic-gate 8960Sstevel@tonic-gate ldxa [%g4]ASI_DTLB_ACCESS, %g5 ! read the data from DTLB_0 8970Sstevel@tonic-gate stx %g5, [%g1 + CH_TLO_TTE_DATA] ! way 0 and store it away 8980Sstevel@tonic-gate ldxa [%g4]ASI_DTLB_TAGREAD, %g5 ! read the tag from DTLB_0 8990Sstevel@tonic-gate stx %g5, [%g1 + CH_TLO_TTE_TAG] ! way 0 and store it away 9000Sstevel@tonic-gate 9010Sstevel@tonic-gate ldxa [%g7]ASI_DTLB_ACCESS, %g5 ! now repeat for DTLB_1 way 0 9020Sstevel@tonic-gate stx %g5, [%g1 + (CH_TLO_TTE_DATA + (CH_TLO_TTE_SIZE * 2))] 9030Sstevel@tonic-gate ldxa [%g7]ASI_DTLB_TAGREAD, %g5 9040Sstevel@tonic-gate stx %g5, [%g1 + (CH_TLO_TTE_TAG + (CH_TLO_TTE_SIZE * 2))] 9050Sstevel@tonic-gate 9060Sstevel@tonic-gate set PN_TLB_ACC_WAY_BIT, %g6 ! same thing again for way 1 9070Sstevel@tonic-gate or %g4, %g6, %g4 ! of each TLB. 9080Sstevel@tonic-gate or %g7, %g6, %g7 9090Sstevel@tonic-gate add %g1, CH_TLO_TTE_SIZE, %g1 ! move up the pointer 9100Sstevel@tonic-gate 9110Sstevel@tonic-gate ldxa [%g4]ASI_DTLB_ACCESS, %g5 ! read the data from DTLB_0 9120Sstevel@tonic-gate stx %g5, [%g1 + CH_TLO_TTE_DATA] ! way 1 and store it away 9130Sstevel@tonic-gate ldxa [%g4]ASI_DTLB_TAGREAD, %g5 ! read the tag from DTLB_0 9140Sstevel@tonic-gate stx %g5, [%g1 + CH_TLO_TTE_TAG] ! way 1 and store it away 9150Sstevel@tonic-gate 9160Sstevel@tonic-gate ldxa [%g7]ASI_DTLB_ACCESS, %g5 ! now repeat for DTLB_1 way 1 9170Sstevel@tonic-gate stx %g5, [%g1 + (CH_TLO_TTE_DATA + (CH_TLO_TTE_SIZE * 2))] 9180Sstevel@tonic-gate ldxa [%g7]ASI_DTLB_TAGREAD, %g5 9190Sstevel@tonic-gate stx %g5, [%g1 + (CH_TLO_TTE_TAG + (CH_TLO_TTE_SIZE * 2))] 9200Sstevel@tonic-gate 9210Sstevel@tonic-gate andn %g4, %g6, %g4 ! back to way 0 9220Sstevel@tonic-gate andn %g7, %g6, %g7 ! back to way 0 9230Sstevel@tonic-gate 9240Sstevel@tonic-gatedtlb_parity_trap_2: 9250Sstevel@tonic-gate /* 9260Sstevel@tonic-gate * at this point: 9270Sstevel@tonic-gate * %g2 - contains the VA whose lookup caused the trap 9280Sstevel@tonic-gate * %g3 - contains the tlo_info field 9290Sstevel@tonic-gate * %g4 - contains the T512_0 access index value for the 9300Sstevel@tonic-gate * VA/PgSz in question 9310Sstevel@tonic-gate * %g7 - contains the T512_1 access index value for the 9320Sstevel@tonic-gate * VA/PgSz in question 9330Sstevel@tonic-gate * 9340Sstevel@tonic-gate * Here we will clear the errors from the DTLB. 9350Sstevel@tonic-gate */ 9360Sstevel@tonic-gate set MMU_TAG_ACCESS, %g5 ! We write a TTE tag value of 9370Sstevel@tonic-gate stxa %g0, [%g5]ASI_DMMU ! 0 as it will be invalid. 9380Sstevel@tonic-gate stxa %g0, [%g4]ASI_DTLB_ACCESS ! Write the data and tag. 9390Sstevel@tonic-gate stxa %g0, [%g7]ASI_DTLB_ACCESS ! Now repeat for DTLB_1 way 0 9400Sstevel@tonic-gate membar #Sync 9410Sstevel@tonic-gate 9420Sstevel@tonic-gate set PN_TLB_ACC_WAY_BIT, %g6 ! same thing again for way 1 9430Sstevel@tonic-gate or %g4, %g6, %g4 9440Sstevel@tonic-gate or %g7, %g6, %g7 9450Sstevel@tonic-gate 9460Sstevel@tonic-gate stxa %g0, [%g4]ASI_DTLB_ACCESS ! Write same data and tag. 9470Sstevel@tonic-gate stxa %g0, [%g7]ASI_DTLB_ACCESS ! Now repeat for DTLB_1 way 0 9480Sstevel@tonic-gate membar #Sync 9490Sstevel@tonic-gate 9500Sstevel@tonic-gate sethi %hi(FLUSH_ADDR), %g6 ! PRM says we need to issue a 9510Sstevel@tonic-gate flush %g6 ! flush after writing MMU regs 9520Sstevel@tonic-gate 9530Sstevel@tonic-gate /* 9540Sstevel@tonic-gate * at this point: 9550Sstevel@tonic-gate * %g2 - contains the VA whose lookup caused the trap 9560Sstevel@tonic-gate * %g3 - contains the tlo_info field 9570Sstevel@tonic-gate * 9580Sstevel@tonic-gate * Call cpu_tlb_parity_error via systrap at PIL 14 unless we're 9590Sstevel@tonic-gate * already at PIL 15. We do this even for TL>1 traps since 9600Sstevel@tonic-gate * those will lead to a system panic. 9610Sstevel@tonic-gate */ 9620Sstevel@tonic-gate set cpu_tlb_parity_error, %g1 9630Sstevel@tonic-gate rdpr %pil, %g4 9640Sstevel@tonic-gate cmp %g4, PIL_14 9650Sstevel@tonic-gate movl %icc, PIL_14, %g4 9660Sstevel@tonic-gate ba sys_trap 9670Sstevel@tonic-gate nop 9680Sstevel@tonic-gate SET_SIZE(dtlb_parity_trap) 9690Sstevel@tonic-gate 9700Sstevel@tonic-gate#endif /* lint */ 9710Sstevel@tonic-gate 9720Sstevel@tonic-gate 9730Sstevel@tonic-gate#if defined(lint) 9740Sstevel@tonic-gate/* 9750Sstevel@tonic-gate * Calculates the Panther TLB index based on a virtual address and page size 9760Sstevel@tonic-gate * 9770Sstevel@tonic-gate * Register usage: 9780Sstevel@tonic-gate * %o0 - virtual address whose index we want 9790Sstevel@tonic-gate * %o1 - Page Size of the TLB in question as encoded in the 9800Sstevel@tonic-gate * ASI_[D|I]MMU_TAG_ACCESS_EXT register. 9810Sstevel@tonic-gate */ 9820Sstevel@tonic-gateuint64_t 9830Sstevel@tonic-gatepn_get_tlb_index(uint64_t va, uint64_t pg_sz) 9840Sstevel@tonic-gate{ 9850Sstevel@tonic-gate return ((va + pg_sz)-(va + pg_sz)); 9860Sstevel@tonic-gate} 9870Sstevel@tonic-gate#else /* lint */ 9880Sstevel@tonic-gate ENTRY(pn_get_tlb_index) 9890Sstevel@tonic-gate 9900Sstevel@tonic-gate PN_GET_TLB_INDEX(%o0, %o1) 9910Sstevel@tonic-gate 9920Sstevel@tonic-gate retl 9930Sstevel@tonic-gate nop 9940Sstevel@tonic-gate SET_SIZE(pn_get_tlb_index) 9950Sstevel@tonic-gate#endif /* lint */ 9960Sstevel@tonic-gate 9970Sstevel@tonic-gate 9980Sstevel@tonic-gate#if defined(lint) 9990Sstevel@tonic-gate/* 10000Sstevel@tonic-gate * For Panther CPUs we need to flush the IPB after any I$ or D$ 10010Sstevel@tonic-gate * parity errors are detected. 10020Sstevel@tonic-gate */ 10030Sstevel@tonic-gatevoid 10040Sstevel@tonic-gateflush_ipb(void) 10050Sstevel@tonic-gate{ return; } 10060Sstevel@tonic-gate 10070Sstevel@tonic-gate#else /* lint */ 10080Sstevel@tonic-gate 10090Sstevel@tonic-gate ENTRY(flush_ipb) 10100Sstevel@tonic-gate clr %o0 10110Sstevel@tonic-gate 10120Sstevel@tonic-gateflush_ipb_1: 10130Sstevel@tonic-gate stxa %g0, [%o0]ASI_IPB_TAG 10140Sstevel@tonic-gate membar #Sync 10150Sstevel@tonic-gate cmp %o0, PN_IPB_TAG_ADDR_MAX 10160Sstevel@tonic-gate blt flush_ipb_1 10170Sstevel@tonic-gate add %o0, PN_IPB_TAG_ADDR_LINESIZE, %o0 10180Sstevel@tonic-gate 10190Sstevel@tonic-gate sethi %hi(FLUSH_ADDR), %o0 10200Sstevel@tonic-gate flush %o0 10210Sstevel@tonic-gate retl 10220Sstevel@tonic-gate nop 10230Sstevel@tonic-gate SET_SIZE(flush_ipb) 10240Sstevel@tonic-gate 10250Sstevel@tonic-gate#endif /* lint */ 10266330Sjc25722 10276330Sjc25722 1028