xref: /onnv-gate/usr/src/uts/sun4u/cpu/spitfire_asm.s (revision 6553:574079756839)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#pragma ident	"%Z%%M%	%I%	%E% SMI"

#if !defined(lint)
#include "assym.h"
#endif	/* lint */

#include <sys/asm_linkage.h>
#include <sys/mmu.h>
#include <vm/hat_sfmmu.h>
#include <sys/machparam.h>
#include <sys/machcpuvar.h>
#include <sys/machthread.h>
#include <sys/privregs.h>
#include <sys/asm_linkage.h>
#include <sys/machasi.h>
#include <sys/trap.h>
#include <sys/spitregs.h>
#include <sys/xc_impl.h>
#include <sys/intreg.h>
#include <sys/async.h>

#ifdef TRAPTRACE
#include <sys/traptrace.h>
#endif /* TRAPTRACE */

#ifndef	lint

/* BEGIN CSTYLED */
#define	DCACHE_FLUSHPAGE(arg1, arg2, tmp1, tmp2, tmp3)			\
	ldxa	[%g0]ASI_LSU, tmp1					;\
	btst	LSU_DC, tmp1		/* is dcache enabled? */	;\
	bz,pn	%icc, 1f						;\
	sethi	%hi(dcache_linesize), tmp1				;\
	ld	[tmp1 + %lo(dcache_linesize)], tmp1			;\
	sethi	%hi(dflush_type), tmp2					;\
	ld	[tmp2 + %lo(dflush_type)], tmp2				;\
	cmp	tmp2, FLUSHPAGE_TYPE					;\
	be,pt	%icc, 2f						;\
	sllx	arg1, SF_DC_VBIT_SHIFT, arg1	/* tag to compare */	;\
	sethi	%hi(dcache_size), tmp3					;\
	ld	[tmp3 + %lo(dcache_size)], tmp3				;\
	cmp	tmp2, FLUSHMATCH_TYPE					;\
	be,pt	%icc, 3f						;\
	nop								;\
	/*								\
	 * flushtype = FLUSHALL_TYPE, flush the whole thing		\
	 * tmp3 = cache size						\
	 * tmp1 = cache line size					\
	 */								\
	sub	tmp3, tmp1, tmp2					;\
4:									\
	stxa	%g0, [tmp2]ASI_DC_TAG					;\
	membar	#Sync							;\
	cmp	%g0, tmp2						;\
	bne,pt	%icc, 4b						;\
	sub	tmp2, tmp1, tmp2					;\
	ba,pt	%icc, 1f						;\
	nop								;\
	/*								\
	 * flushtype = FLUSHPAGE_TYPE					\
	 * arg1 = tag to compare against				\
	 * arg2 = virtual color						\
	 * tmp1 = cache line size					\
	 * tmp2 = tag from cache					\
	 * tmp3 = counter						\
	 */								\
2:									\
	set	MMU_PAGESIZE, tmp3					;\
	sllx	arg2, MMU_PAGESHIFT, arg2  /* color to dcache page */	;\
	sub	tmp3, tmp1, tmp3					;\
4:									\
	ldxa	[arg2 + tmp3]ASI_DC_TAG, tmp2	/* read tag */		;\
	btst	SF_DC_VBIT_MASK, tmp2					;\
	bz,pn	%icc, 5f	  /* branch if no valid sub-blocks */	;\
	andn	tmp2, SF_DC_VBIT_MASK, tmp2	/* clear out v bits */	;\
	cmp	tmp2, arg1						;\
	bne,pn	%icc, 5f			/* br if tag miss */	;\
	nop								;\
	stxa	%g0, [arg2 + tmp3]ASI_DC_TAG				;\
	membar	#Sync							;\
5:									\
	cmp	%g0, tmp3						;\
	bnz,pt	%icc, 4b		/* branch if not done */	;\
	sub	tmp3, tmp1, tmp3					;\
	ba,pt	%icc, 1f						;\
	nop								;\
	/*								\
	 * flushtype = FLUSHMATCH_TYPE					\
	 * arg1 = tag to compare against				\
	 * tmp1 = cache line size					\
	 * tmp3 = cache size						\
	 * arg2 = counter						\
	 * tmp2 = cache tag						\
	 */								\
3:									\
	sub	tmp3, tmp1, arg2					;\
4:									\
	ldxa	[arg2]ASI_DC_TAG, tmp2		/* read tag */		;\
	btst	SF_DC_VBIT_MASK, tmp2					;\
	bz,pn	%icc, 5f		/* br if no valid sub-blocks */	;\
	andn	tmp2, SF_DC_VBIT_MASK, tmp2	/* clear out v bits */	;\
	cmp	tmp2, arg1						;\
	bne,pn	%icc, 5f		/* branch if tag miss */	;\
	nop								;\
	stxa	%g0, [arg2]ASI_DC_TAG					;\
	membar	#Sync							;\
5:									\
	cmp	%g0, arg2						;\
	bne,pt	%icc, 4b		/* branch if not done */	;\
	sub	arg2, tmp1, arg2					;\
1:

/*
 * macro that flushes the entire dcache color
 */
#define	DCACHE_FLUSHCOLOR(arg, tmp1, tmp2)				\
	ldxa	[%g0]ASI_LSU, tmp1;					\
	btst	LSU_DC, tmp1;		/* is dcache enabled? */	\
	bz,pn	%icc, 1f;						\
	sethi	%hi(dcache_linesize), tmp1;				\
	ld	[tmp1 + %lo(dcache_linesize)], tmp1;			\
	set	MMU_PAGESIZE, tmp2;					\
	/*								\
	 * arg = virtual color						\
	 * tmp2 = page size						\
	 * tmp1 = cache line size					\
	 */								\
	sllx	arg, MMU_PAGESHIFT, arg; /* color to dcache page */	\
	sub	tmp2, tmp1, tmp2;					\
2:									\
	stxa	%g0, [arg + tmp2]ASI_DC_TAG;				\
	membar	#Sync;							\
	cmp	%g0, tmp2;						\
	bne,pt	%icc, 2b;						\
	sub	tmp2, tmp1, tmp2;					\
1:

/*
 * macro that flushes the entire dcache
 */
#define	DCACHE_FLUSHALL(size, linesize, tmp)				\
	ldxa	[%g0]ASI_LSU, tmp;					\
	btst	LSU_DC, tmp;		/* is dcache enabled? */	\
	bz,pn	%icc, 1f;						\
									\
	sub	size, linesize, tmp;					\
2:									\
	stxa	%g0, [tmp]ASI_DC_TAG;					\
	membar	#Sync;							\
	cmp	%g0, tmp;						\
	bne,pt	%icc, 2b;						\
	sub	tmp, linesize, tmp;					\
1:

/*
 * macro that flushes the entire icache
 */
#define	ICACHE_FLUSHALL(size, linesize, tmp)				\
	ldxa	[%g0]ASI_LSU, tmp;					\
	btst	LSU_IC, tmp;						\
	bz,pn	%icc, 1f;						\
									\
	sub	size, linesize, tmp;					\
2:									\
	stxa	%g0, [tmp]ASI_IC_TAG;					\
	membar	#Sync;							\
	cmp	%g0, tmp;						\
	bne,pt	%icc, 2b;						\
	sub	tmp, linesize, tmp;					\
1:

#ifdef SF_ERRATA_32
#define SF_WORKAROUND(tmp1, tmp2)                               \
        sethi   %hi(FLUSH_ADDR), tmp2                           ;\
        set     MMU_PCONTEXT, tmp1                              ;\
        stxa    %g0, [tmp1]ASI_DMMU                             ;\
        flush   tmp2                                            ;
#else
#define SF_WORKAROUND(tmp1, tmp2)
#endif /* SF_ERRATA_32 */

/*
 * arg1 = vaddr
 * arg2 = ctxnum
 *      - disable interrupts and clear address mask
 *        to access 64 bit physaddr
 *      - Blow out the TLB, flush user page.
 *        . use secondary context.
 */
#define VTAG_FLUSHUPAGE(lbl, arg1, arg2, tmp1, tmp2, tmp3, tmp4) \
        rdpr    %pstate, tmp1                                   ;\
        andn    tmp1, PSTATE_IE, tmp2				;\
        wrpr    tmp2, 0, %pstate                                ;\
        sethi   %hi(FLUSH_ADDR), tmp2                           ;\
        set     MMU_SCONTEXT, tmp3                              ;\
        ldxa    [tmp3]ASI_DMMU, tmp4                            ;\
        or      DEMAP_SECOND | DEMAP_PAGE_TYPE, arg1, arg1      ;\
        cmp     tmp4, arg2                                      ;\
        be,a,pt %icc, lbl/**/4                                  ;\
          nop                                                   ;\
        stxa    arg2, [tmp3]ASI_DMMU                            ;\
lbl/**/4:                                                       ;\
        stxa    %g0, [arg1]ASI_DTLB_DEMAP                       ;\
        stxa    %g0, [arg1]ASI_ITLB_DEMAP                       ;\
        flush   tmp2                                            ;\
        be,a,pt %icc, lbl/**/5                                  ;\
          nop                                                   ;\
        stxa    tmp4, [tmp3]ASI_DMMU                            ;\
        flush   tmp2                                            ;\
lbl/**/5:                                                       ;\
        wrpr    %g0, tmp1, %pstate


/*
 * macro that flushes all the user entries in dtlb
 * arg1 = dtlb entries
 *	- Before first compare:
 *              tmp4 = tte
 *              tmp5 = vaddr
 *              tmp6 = cntxnum
 */
#define DTLB_FLUSH_UNLOCKED_UCTXS(lbl, arg1, tmp1, tmp2, tmp3, \
                                tmp4, tmp5, tmp6) \
lbl/**/0:                                                       ;\
        sllx    arg1, 3, tmp3                                   ;\
        SF_WORKAROUND(tmp1, tmp2)                               ;\
        ldxa    [tmp3]ASI_DTLB_ACCESS, tmp4                     ;\
        srlx    tmp4, 6, tmp4                                   ;\
        andcc   tmp4, 1, %g0                                    ;\
        bnz,pn  %xcc, lbl/**/1                                  ;\
        srlx    tmp4, 57, tmp4                                  ;\
        andcc   tmp4, 1, %g0                                    ;\
        beq,pn  %xcc, lbl/**/1                                  ;\
          nop                                                   ;\
        set     TAGREAD_CTX_MASK, tmp1                          ;\
        ldxa    [tmp3]ASI_DTLB_TAGREAD, tmp2                    ;\
        and     tmp2, tmp1, tmp6                                ;\
        andn    tmp2, tmp1, tmp5                                ;\
	set	KCONTEXT, tmp4					;\
	cmp	tmp6, tmp4					;\
	be	lbl/**/1					;\
	  nop							;\
        VTAG_FLUSHUPAGE(VD/**/lbl, tmp5, tmp6, tmp1, tmp2, tmp3, tmp4) ;\
lbl/**/1:                                                       ;\
        brgz,pt arg1, lbl/**/0                                  ;\
          sub     arg1, 1, arg1


/*
 * macro that flushes all the user entries in itlb
 * arg1 = itlb entries
 *      - Before first compare:
 *              tmp4 = tte
 *              tmp5 = vaddr
 *              tmp6 = cntxnum
 */
#define ITLB_FLUSH_UNLOCKED_UCTXS(lbl, arg1, tmp1, tmp2, tmp3, \
                                tmp4, tmp5, tmp6) \
lbl/**/0:                                                       ;\
        sllx    arg1, 3, tmp3                                   ;\
        SF_WORKAROUND(tmp1, tmp2)                               ;\
        ldxa    [tmp3]ASI_ITLB_ACCESS, tmp4                     ;\
        srlx    tmp4, 6, tmp4                                   ;\
        andcc   tmp4, 1, %g0                                    ;\
        bnz,pn  %xcc, lbl/**/1                                  ;\
        srlx    tmp4, 57, tmp4                                  ;\
        andcc   tmp4, 1, %g0                                    ;\
        beq,pn  %xcc, lbl/**/1                                  ;\
          nop                                                   ;\
        set     TAGREAD_CTX_MASK, tmp1                          ;\
        ldxa    [tmp3]ASI_ITLB_TAGREAD, tmp2                    ;\
        and     tmp2, tmp1, tmp6                                ;\
        andn    tmp2, tmp1, tmp5                                ;\
	set	KCONTEXT, tmp4					;\
	cmp	tmp6, tmp4					;\
	be	lbl/**/1					;\
	  nop							;\
        VTAG_FLUSHUPAGE(VI/**/lbl, tmp5, tmp6, tmp1, tmp2, tmp3, tmp4) ;\
lbl/**/1:                                                       ;\
        brgz,pt arg1, lbl/**/0                                  ;\
        sub     arg1, 1, arg1



/*
 * Macro for getting to offset from 'cpu_private' ptr. The 'cpu_private'
 * ptr is in the machcpu structure.
 * r_or_s:	Register or symbol off offset from 'cpu_private' ptr.
 * scr1:	Scratch, ptr is returned in this register.
 * scr2:	Scratch
 */
#define GET_CPU_PRIVATE_PTR(r_or_s, scr1, scr2, label)		\
	CPU_ADDR(scr1, scr2);						\
	ldn	[scr1 + CPU_PRIVATE], scr1; 				\
	cmp	scr1, 0; 						\
	be	label;							\
	 nop; 								\
	add	scr1, r_or_s, scr1;  					\

#ifdef HUMMINGBIRD
/*
 * UltraSPARC-IIe processor supports both 4-way set associative and
 * direct map E$. For performance reasons, we flush E$ by placing it
 * in direct map mode for data load/store and restore the state after
 * we are done flushing it. Keep interrupts off while flushing in this
 * manner.
 *
 * We flush the entire ecache by starting at one end and loading each
 * successive ecache line for the 2*ecache-size range. We have to repeat
 * the flush operation to guarantee that the entire ecache has been
 * flushed.
 *
 * For flushing a specific physical address, we start at the aliased
 * address and load at set-size stride, wrapping around at 2*ecache-size
 * boundary and skipping the physical address being flushed. It takes
 * 10 loads to guarantee that the physical address has been flushed.
 */

#define	HB_ECACHE_FLUSH_CNT	2
#define	HB_PHYS_FLUSH_CNT	10	/* #loads to flush specific paddr */
#endif /* HUMMINGBIRD */

/* END CSTYLED */

#endif	/* !lint */

/*
 * Spitfire MMU and Cache operations.
 */

#if defined(lint)

/*ARGSUSED*/
void
vtag_flushpage(caddr_t vaddr, uint64_t sfmmup)
{}

/*ARGSUSED*/
void
vtag_flushall(void)
{}

/*ARGSUSED*/
void
vtag_flushall_uctxs(void)
{}

/*ARGSUSED*/
void
vtag_flushpage_tl1(uint64_t vaddr, uint64_t sfmmup)
{}

/*ARGSUSED*/
void
vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t sfmmup_pgcnt)
{}

/*ARGSUSED*/
void
vtag_flushall_tl1(uint64_t dummy1, uint64_t dummy2)
{}

/*ARGSUSED*/
void
vac_flushpage(pfn_t pfnum, int vcolor)
{}

/*ARGSUSED*/
void
vac_flushpage_tl1(uint64_t pfnum, uint64_t vcolor)
{}

/*ARGSUSED*/
void
init_mondo(xcfunc_t *func, uint64_t arg1, uint64_t arg2)
{}

/*ARGSUSED*/
void
init_mondo_nocheck(xcfunc_t *func, uint64_t arg1, uint64_t arg2)
{}

/*ARGSUSED*/
void
flush_instr_mem(caddr_t vaddr, size_t len)
{}

/*ARGSUSED*/
void
flush_ecache(uint64_t physaddr, size_t size, size_t linesize)
{}

/*ARGSUSED*/
void
get_ecache_dtag(uint32_t ecache_idx, uint64_t *ecache_data,
		uint64_t *ecache_tag, uint64_t *oafsr, uint64_t *acc_afsr)
{}

/* ARGSUSED */
uint64_t
get_ecache_tag(uint32_t id, uint64_t *nafsr, uint64_t *acc_afsr)
{
	return ((uint64_t)0);
}

/* ARGSUSED */
uint64_t
check_ecache_line(uint32_t id, uint64_t *acc_afsr)
{
	return ((uint64_t)0);
}

/*ARGSUSED*/
void
kdi_flush_idcache(int dcache_size, int dcache_lsize,
    int icache_size, int icache_lsize)
{}

#else	/* lint */

	ENTRY_NP(vtag_flushpage)
	/*
	 * flush page from the tlb
	 *
	 * %o0 = vaddr
	 * %o1 = sfmmup
	 */
	rdpr	%pstate, %o5
#ifdef DEBUG
	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfdi_label1, %g1)
#endif /* DEBUG */
	/*
	 * disable ints
	 */
	andn	%o5, PSTATE_IE, %o4
	wrpr	%o4, 0, %pstate

	/*
	 * Then, blow out the tlb
	 * Interrupts are disabled to prevent the secondary ctx register
	 * from changing underneath us.
	 */
	sethi   %hi(ksfmmup), %o3
        ldx     [%o3 + %lo(ksfmmup)], %o3
        cmp     %o3, %o1
        bne,pt   %xcc, 1f			! if not kernel as, go to 1
	  sethi	%hi(FLUSH_ADDR), %o3
	/*
	 * For KCONTEXT demaps use primary. type = page implicitly
	 */
	stxa	%g0, [%o0]ASI_DTLB_DEMAP	/* dmmu flush for KCONTEXT */
	stxa	%g0, [%o0]ASI_ITLB_DEMAP	/* immu flush for KCONTEXT */
	flush	%o3
	b	5f
	  nop
1:
	/*
	 * User demap.  We need to set the secondary context properly.
	 * %o0 = vaddr
	 * %o1 = sfmmup
	 * %o3 = FLUSH_ADDR
	 */
	SFMMU_CPU_CNUM(%o1, %g1, %g2)	/* %g1 = sfmmu cnum on this CPU */

	set	MMU_SCONTEXT, %o4
	ldxa	[%o4]ASI_DMMU, %o2		/* rd old ctxnum */
	or	DEMAP_SECOND | DEMAP_PAGE_TYPE, %o0, %o0
	cmp	%o2, %g1
	be,pt	%icc, 4f
	  nop
	stxa	%g1, [%o4]ASI_DMMU		/* wr new ctxum */
4:
	stxa	%g0, [%o0]ASI_DTLB_DEMAP
	stxa	%g0, [%o0]ASI_ITLB_DEMAP
	flush	%o3
	be,pt	%icc, 5f
	  nop
	stxa	%o2, [%o4]ASI_DMMU		/* restore old ctxnum */
	flush	%o3
5:
	retl
	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
	SET_SIZE(vtag_flushpage)

        .seg    ".text"
.flushallmsg:
        .asciz  "sfmmu_asm: unimplemented flush operation"

        ENTRY_NP(vtag_flushall)
        sethi   %hi(.flushallmsg), %o0
        call    panic
          or    %o0, %lo(.flushallmsg), %o0
        SET_SIZE(vtag_flushall)

	ENTRY_NP(vtag_flushall_uctxs)
	/*
	 * flush entire DTLB/ITLB.
	 */
	CPU_INDEX(%g1, %g2)
	mulx	%g1, CPU_NODE_SIZE, %g1
	set	cpunodes, %g2
	add	%g1, %g2, %g1
	lduh	[%g1 + ITLB_SIZE], %g2		! %g2 = # entries in ITLB
	lduh	[%g1 + DTLB_SIZE], %g1		! %g1 = # entries in DTLB
	sub	%g2, 1, %g2			! %g2 = # entries in ITLB - 1
	sub	%g1, 1, %g1			! %g1 = # entries in DTLB - 1

        !
        ! Flush itlb's
        !
        ITLB_FLUSH_UNLOCKED_UCTXS(I, %g2, %g3, %g4, %o2, %o3, %o4, %o5)

	!
        ! Flush dtlb's
        !
        DTLB_FLUSH_UNLOCKED_UCTXS(D, %g1, %g3, %g4, %o2, %o3, %o4, %o5)

	membar  #Sync
	retl
	  nop

	SET_SIZE(vtag_flushall_uctxs)

	ENTRY_NP(vtag_flushpage_tl1)
	/*
	 * x-trap to flush page from tlb and tsb
	 *
	 * %g1 = vaddr, zero-extended on 32-bit kernel
	 * %g2 = sfmmup
	 *
	 * assumes TSBE_TAG = 0
	 */
	srln	%g1, MMU_PAGESHIFT, %g1
	slln	%g1, MMU_PAGESHIFT, %g1			/* g1 = vaddr */

	SFMMU_CPU_CNUM(%g2, %g3, %g4)   /* %g3 = sfmmu cnum on this CPU */

	/* We need to set the secondary context properly. */
	set	MMU_SCONTEXT, %g4
	ldxa	[%g4]ASI_DMMU, %g5		/* rd old ctxnum */
	or	DEMAP_SECOND | DEMAP_PAGE_TYPE, %g1, %g1
	stxa	%g3, [%g4]ASI_DMMU		/* wr new ctxum */
	stxa	%g0, [%g1]ASI_DTLB_DEMAP
	stxa	%g0, [%g1]ASI_ITLB_DEMAP
	stxa	%g5, [%g4]ASI_DMMU		/* restore old ctxnum */
	membar #Sync
	retry
	SET_SIZE(vtag_flushpage_tl1)

	ENTRY_NP(vtag_flush_pgcnt_tl1)
	/*
	 * x-trap to flush pgcnt MMU_PAGESIZE pages from tlb
	 *
	 * %g1 = vaddr, zero-extended on 32-bit kernel
	 * %g2 = <sfmmup58 | pgcnt6>
	 *
	 * NOTE: this handler relies on the fact that no
	 *	interrupts or traps can occur during the loop
	 *	issuing the TLB_DEMAP operations. It is assumed
	 *	that interrupts are disabled and this code is
	 *	fetching from the kernel locked text address.
	 *
	 * assumes TSBE_TAG = 0
	 */
	srln	%g1, MMU_PAGESHIFT, %g1
	slln	%g1, MMU_PAGESHIFT, %g1		/* g1 = vaddr */
	or	DEMAP_SECOND | DEMAP_PAGE_TYPE, %g1, %g1

	set	SFMMU_PGCNT_MASK, %g4
	and	%g4, %g2, %g3			/* g3 = pgcnt - 1 */
	add	%g3, 1, %g3			/* g3 = pgcnt */

	andn	%g2, SFMMU_PGCNT_MASK, %g2	/* g2 = sfmmup */

	SFMMU_CPU_CNUM(%g2, %g5, %g6)   ! %g5 = sfmmu cnum on this CPU

	/* We need to set the secondary context properly. */
	set	MMU_SCONTEXT, %g4
	ldxa	[%g4]ASI_DMMU, %g6		/* read old ctxnum */
	stxa	%g5, [%g4]ASI_DMMU		/* write new ctxum */

	set	MMU_PAGESIZE, %g2		/* g2 = pgsize */
	sethi	 %hi(FLUSH_ADDR), %g5
1:
	stxa	%g0, [%g1]ASI_DTLB_DEMAP
	stxa	%g0, [%g1]ASI_ITLB_DEMAP
	flush	%g5
	deccc	%g3				/* decr pgcnt */
	bnz,pt	%icc,1b
	  add	%g1, %g2, %g1			/* go to nextpage */

	stxa	%g6, [%g4]ASI_DMMU		/* restore old ctxnum */
	membar #Sync
	retry
	SET_SIZE(vtag_flush_pgcnt_tl1)

	! Not implemented on US1/US2
	ENTRY_NP(vtag_flushall_tl1)
	retry
	SET_SIZE(vtag_flushall_tl1)

/*
 * vac_flushpage(pfnum, color)
 *	Flush 1 8k page of the D-$ with physical page = pfnum
 *	Algorithm:
 *		The spitfire dcache is a 16k direct mapped virtual indexed,
 *		physically tagged cache.  Given the pfnum we read all cache
 *		lines for the corresponding page in the cache (determined by
 *		the color).  Each cache line is compared with
 *		the tag created from the pfnum. If the tags match we flush
 *		the line.
 */
	.seg	".data"
	.align	8
	.global	dflush_type
dflush_type:
	.word	FLUSHPAGE_TYPE
	.seg	".text"

	ENTRY(vac_flushpage)
	/*
	 * flush page from the d$
	 *
	 * %o0 = pfnum, %o1 = color
	 */
	DCACHE_FLUSHPAGE(%o0, %o1, %o2, %o3, %o4)
	retl
	nop
	SET_SIZE(vac_flushpage)

	ENTRY_NP(vac_flushpage_tl1)
	/*
	 * x-trap to flush page from the d$
	 *
	 * %g1 = pfnum, %g2 = color
	 */
	DCACHE_FLUSHPAGE(%g1, %g2, %g3, %g4, %g5)
	retry
	SET_SIZE(vac_flushpage_tl1)

	ENTRY(vac_flushcolor)
	/*
	 * %o0 = vcolor
	 */
	DCACHE_FLUSHCOLOR(%o0, %o1, %o2)
	retl
	  nop
	SET_SIZE(vac_flushcolor)

	ENTRY(vac_flushcolor_tl1)
	/*
	 * %g1 = vcolor
	 */
	DCACHE_FLUSHCOLOR(%g1, %g2, %g3)
	retry
	SET_SIZE(vac_flushcolor_tl1)


	.global _dispatch_status_busy
_dispatch_status_busy:
	.asciz	"ASI_INTR_DISPATCH_STATUS error: busy"
	.align	4

/*
 * Determine whether or not the IDSR is busy.
 * Entry: no arguments
 * Returns: 1 if busy, 0 otherwise
 */
	ENTRY(idsr_busy)
	ldxa	[%g0]ASI_INTR_DISPATCH_STATUS, %g1
	clr	%o0
	btst	IDSR_BUSY, %g1
	bz,a,pt	%xcc, 1f
	mov	1, %o0
1:
	retl
	nop
	SET_SIZE(idsr_busy)

/*
 * Setup interrupt dispatch data registers
 * Entry:
 *	%o0 - function or inumber to call
 *	%o1, %o2 - arguments (2 uint64_t's)
 */
	.seg "text"

	ENTRY(init_mondo)
#ifdef DEBUG
	!
	! IDSR should not be busy at the moment
	!
	ldxa	[%g0]ASI_INTR_DISPATCH_STATUS, %g1
	btst	IDSR_BUSY, %g1
	bz,pt	%xcc, 1f
	nop

	sethi	%hi(_dispatch_status_busy), %o0
	call	panic
	or	%o0, %lo(_dispatch_status_busy), %o0
#endif /* DEBUG */

	ALTENTRY(init_mondo_nocheck)
	!
	! interrupt vector dispach data reg 0
	!
1:
	mov	IDDR_0, %g1
	mov	IDDR_1, %g2
	mov	IDDR_2, %g3
	stxa	%o0, [%g1]ASI_INTR_DISPATCH

	!
	! interrupt vector dispach data reg 1
	!
	stxa	%o1, [%g2]ASI_INTR_DISPATCH

	!
	! interrupt vector dispach data reg 2
	!
	stxa	%o2, [%g3]ASI_INTR_DISPATCH

	retl
	membar	#Sync			! allowed to be in the delay slot
	SET_SIZE(init_mondo)

/*
 * Ship mondo to upaid
 */
	ENTRY_NP(shipit)
	sll	%o0, IDCR_PID_SHIFT, %g1	! IDCR<18:14> = upa id
	or	%g1, IDCR_OFFSET, %g1		! IDCR<13:0> = 0x70
	stxa	%g0, [%g1]ASI_INTR_DISPATCH	! interrupt vector dispatch
#if defined(SF_ERRATA_54)
	membar	#Sync				! store must occur before load
	mov	0x20, %g3			! UDBH Control Register Read
	ldxa	[%g3]ASI_SDB_INTR_R, %g0
#endif
	retl
	membar	#Sync
	SET_SIZE(shipit)


/*
 * flush_instr_mem:
 *	Flush a portion of the I-$ starting at vaddr
 * 	%o0 vaddr
 *	%o1 bytes to be flushed
 */

	ENTRY(flush_instr_mem)
	membar	#StoreStore				! Ensure the stores
							! are globally visible
1:
	flush	%o0
	subcc	%o1, ICACHE_FLUSHSZ, %o1		! bytes = bytes-0x20
	bgu,pt	%ncc, 1b
	add	%o0, ICACHE_FLUSHSZ, %o0		! vaddr = vaddr+0x20

	retl
	nop
	SET_SIZE(flush_instr_mem)

/*
 * flush_ecache:
 * Flush the entire e$ using displacement flush by reading through a
 * physically contiguous area. We use mmu bypass asi (ASI_MEM) while
 * reading this physical address range so that data doesn't go to d$.
 * incoming arguments:
 *	%o0 - 64 bit physical address
 *	%o1 - size of address range to read
 *	%o2 - ecache linesize
 */
	ENTRY(flush_ecache)
#ifndef HUMMINGBIRD
	b	2f
	  nop
1:
	ldxa	[%o0 + %o1]ASI_MEM, %g0	! start reading from physaddr + size
2:
	subcc	%o1, %o2, %o1
	bcc,a,pt %ncc, 1b
	  nop

#else /* HUMMINGBIRD */
	/*
	 * UltraSPARC-IIe processor supports both 4-way set associative
	 * and direct map E$. For performance reasons, we flush E$ by
	 * placing it in direct map mode for data load/store and restore
	 * the state after we are done flushing it. It takes 2 iterations
	 * to guarantee that the entire ecache has been flushed.
	 *
	 * Keep the interrupts disabled while flushing E$ in this manner.
	 */
	rdpr	%pstate, %g4		! current pstate (restored later)
	andn	%g4, PSTATE_IE, %g5
	wrpr	%g0, %g5, %pstate	! disable interrupts

	! Place E$ in direct map mode for data access
	or	%g0, 1, %g5
	sllx	%g5, HB_UPA_DMAP_DATA_BIT, %g5
	ldxa	[%g0]ASI_UPA_CONFIG, %g1 ! current UPA config (restored later)
	or	%g1, %g5, %g5
	membar	#Sync
	stxa	%g5, [%g0]ASI_UPA_CONFIG ! enable direct map for data access
	membar	#Sync

	! flush entire ecache HB_ECACHE_FLUSH_CNT times
	mov	HB_ECACHE_FLUSH_CNT-1, %g5
2:
	sub	%o1, %o2, %g3		! start from last entry
1:
	ldxa	[%o0 + %g3]ASI_MEM, %g0	! start reading from physaddr + size
	subcc	%g3, %o2, %g3
	bgeu,a,pt %ncc, 1b
	  nop
	brgz,a,pt %g5, 2b
	  dec	%g5

	membar	#Sync
	stxa	%g1, [%g0]ASI_UPA_CONFIG ! restore UPA config reg
	membar	#Sync
	wrpr	%g0, %g4, %pstate	! restore earlier pstate
#endif /* HUMMINGBIRD */

	retl
	nop
	SET_SIZE(flush_ecache)

/*
 * void kdi_flush_idcache(int dcache_size, int dcache_linesize,
 *			int icache_size, int icache_linesize)
 */
	ENTRY(kdi_flush_idcache)
	DCACHE_FLUSHALL(%o0, %o1, %g1)
	ICACHE_FLUSHALL(%o2, %o3, %g1)
	membar	#Sync
	retl
	nop
	SET_SIZE(kdi_flush_idcache)


/*
 * void get_ecache_dtag(uint32_t ecache_idx, uint64_t *data, uint64_t *tag,
 * 			uint64_t *oafsr, uint64_t *acc_afsr)
 *
 * Get ecache data and tag.  The ecache_idx argument is assumed to be aligned
 * on a 64-byte boundary.  The corresponding AFSR value is also read for each
 * 8 byte ecache data obtained. The ecache data is assumed to be a pointer
 * to an array of 16 uint64_t's (e$data & afsr value).  The action to read the
 * data and tag should be atomic to make sense.  We will be executing at PIL15
 * and will disable IE, so nothing can occur between the two reads.  We also
 * assume that the execution of this code does not interfere with what we are
 * reading - not really possible, but we'll live with it for now.
 * We also pass the old AFSR value before clearing it, and caller will take
 * appropriate actions if the important bits are non-zero.
 *
 * If the caller wishes to track the AFSR in cases where the CP bit is
 * set, an address should be passed in for acc_afsr.  Otherwise, this
 * argument may be null.
 *
 * Register Usage:
 * i0: In: 32-bit e$ index
 * i1: In: addr of e$ data
 * i2: In: addr of e$ tag
 * i3: In: addr of old afsr
 * i4: In: addr of accumulated afsr - may be null
 */
	ENTRY(get_ecache_dtag)
	save	%sp, -SA(MINFRAME), %sp
	or	%g0, 1, %l4
	sllx	%l4, 39, %l4	! set bit 39 for e$ data access
	or	%i0, %l4, %g6	! %g6 = e$ addr for data read
	sllx	%l4, 1, %l4	! set bit 40 for e$ tag access
	or	%i0, %l4, %l4	! %l4 = e$ addr for tag read

	rdpr    %pstate, %i5
	andn    %i5, PSTATE_IE | PSTATE_AM, %i0
	wrpr    %i0, %g0, %pstate       ! clear IE, AM bits

	ldxa    [%g0]ASI_ESTATE_ERR, %g1
	stxa    %g0, [%g0]ASI_ESTATE_ERR        ! disable errors
	membar  #Sync

	ldxa	[%g0]ASI_AFSR, %i0      ! grab the old-afsr before tag read
	stx     %i0, [%i3]		! write back the old-afsr

	ldxa    [%l4]ASI_EC_R, %g0      ! read tag into E$ tag reg
	ldxa    [%g0]ASI_EC_DIAG, %i0   ! read tag from E$ tag reg
	stx     %i0, [%i2]              ! write back tag result

	clr	%i2			! loop count

	brz	%i4, 1f			! acc_afsr == NULL?
	  ldxa	[%g0]ASI_AFSR, %i0      ! grab the old-afsr before clearing
	srlx	%i0, P_AFSR_CP_SHIFT, %l0
	btst	1, %l0
	bz	1f
	  nop
	ldx	[%i4], %g4
	or	%g4, %i0, %g4		! aggregate AFSR in cpu private
	stx	%g4, [%i4]
1:
	stxa    %i0, [%g0]ASI_AFSR	! clear AFSR
	membar  #Sync
	ldxa    [%g6]ASI_EC_R, %i0      ! read the 8byte E$data
	stx     %i0, [%i1]              ! save the E$data
	add     %g6, 8, %g6
	add     %i1, 8, %i1
	ldxa    [%g0]ASI_AFSR, %i0      ! read AFSR for this 16byte read
	srlx	%i0, P_AFSR_CP_SHIFT, %l0
	btst	1, %l0
	bz	2f
	  stx     %i0, [%i1]		! save the AFSR

	brz	%i4, 2f			! acc_afsr == NULL?
	  nop
	ldx	[%i4], %g4
	or	%g4, %i0, %g4		! aggregate AFSR in cpu private
	stx	%g4, [%i4]
2:
	add     %i2, 8, %i2
	cmp     %i2, 64
	bl,a    1b
	  add     %i1, 8, %i1
	stxa    %i0, [%g0]ASI_AFSR              ! clear AFSR
	membar  #Sync
	stxa    %g1, [%g0]ASI_ESTATE_ERR        ! restore error enable
	membar  #Sync
	wrpr    %g0, %i5, %pstate
	ret
	  restore
	SET_SIZE(get_ecache_dtag)
#endif /* lint */

#if defined(lint)
/*
 * The ce_err function handles trap type 0x63 (corrected_ECC_error) at tl=0.
 * Steps: 1. GET AFSR  2. Get AFAR <40:4> 3. Get datapath error status
 *	  4. Clear datapath error bit(s) 5. Clear AFSR error bit
 *	  6. package data in %g2 and %g3 7. call cpu_ce_error vis sys_trap
 * %g2: [ 52:43 UDB lower | 42:33 UDB upper | 32:0 afsr ] - arg #3/arg #1
 * %g3: [ 40:4 afar ] - sys_trap->have_win: arg #4/arg #2
 */
void
ce_err(void)
{}

void
ce_err_tl1(void)
{}


/*
 * The async_err function handles trap types 0x0A (instruction_access_error)
 * and 0x32 (data_access_error) at TL = 0 and TL > 0.  When we branch here,
 * %g5 will have the trap type (with 0x200 set if we're at TL > 0).
 *
 * Steps: 1. Get AFSR 2. Get AFAR <40:4> 3. If not UE error skip UDP registers.
 *	  4. Else get and clear datapath error bit(s) 4. Clear AFSR error bits
 *	  6. package data in %g2 and %g3 7. disable all cpu errors, because
 *	  trap is likely to be fatal 8. call cpu_async_error vis sys_trap
 *
 * %g3: [ 63:53 tt | 52:43 UDB_L | 42:33 UDB_U | 32:0 afsr ] - arg #3/arg #1
 * %g2: [ 40:4 afar ] - sys_trap->have_win: arg #4/arg #2
 */
void
async_err(void)
{}

/*
 * The clr_datapath function clears any error bits set in the UDB regs.
 */
void
clr_datapath(void)
{}

/*
 * The get_udb_errors() function gets the current value of the
 * Datapath Error Registers.
 */
/*ARGSUSED*/
void
get_udb_errors(uint64_t *udbh, uint64_t *udbl)
{
	*udbh = 0;
	*udbl = 0;
}

#else 	/* lint */

	ENTRY_NP(ce_err)
	ldxa	[%g0]ASI_AFSR, %g3	! save afsr in g3

	!
	! Check for a UE... From Kevin.Normoyle:
	! We try to switch to the trap for the UE, but since that's
	! a hardware pipeline, we might get to the CE trap before we
	! can switch. The UDB and AFSR registers will have both the
	! UE and CE bits set but the UDB syndrome and the AFAR will be
	! for the UE.
	!
	or	%g0, 1, %g1		! put 1 in g1
	sllx	%g1, 21, %g1		! shift left to <21> afsr UE
	andcc	%g1, %g3, %g0		! check for UE in afsr
	bnz	async_err		! handle the UE, not the CE
	  or	%g0, 0x63, %g5		! pass along the CE ttype
	!
	! Disable further CE traps to avoid recursion (stack overflow)
	! and staying above XCALL_PIL for extended periods.
	!
	ldxa	[%g0]ASI_ESTATE_ERR, %g2
	andn	%g2, 0x1, %g2		! clear bit 0 - CEEN
	stxa	%g2, [%g0]ASI_ESTATE_ERR
	membar	#Sync			! required
	!
	! handle the CE
	ldxa	[%g0]ASI_AFAR, %g2	! save afar in g2

	set	P_DER_H, %g4		! put P_DER_H in g4
	ldxa	[%g4]ASI_SDB_INTR_R, %g5 ! read sdb upper half into g5
	or	%g0, 1, %g6		! put 1 in g6
	sllx	%g6, 8, %g6		! shift g6 to <8> sdb CE
	andcc	%g5, %g6, %g1		! check for CE in upper half
	sllx	%g5, 33, %g5		! shift upper bits to <42:33>
	or	%g3, %g5, %g3		! or with afsr bits
	bz,a	1f			! no error, goto 1f
	  nop
	stxa	%g1, [%g4]ASI_SDB_INTR_W ! clear sdb reg error bit
	membar	#Sync			! membar sync required
1:
	set	P_DER_L, %g4		! put P_DER_L in g4
	ldxa	[%g4]ASI_SDB_INTR_R, %g5 ! read sdb lower half into g6
	andcc	%g5, %g6, %g1		! check for CE in lower half
	sllx	%g5, 43, %g5		! shift upper bits to <52:43>
	or	%g3, %g5, %g3		! or with afsr bits
	bz,a	2f			! no error, goto 2f
	  nop
	stxa	%g1, [%g4]ASI_SDB_INTR_W ! clear sdb reg error bit
	membar	#Sync			! membar sync required
2:
	or	%g0, 1, %g4		! put 1 in g4
	sllx	%g4, 20, %g4		! shift left to <20> afsr CE
	stxa	%g4, [%g0]ASI_AFSR	! use g4 to clear afsr CE error
	membar	#Sync			! membar sync required

	set	cpu_ce_error, %g1	! put *cpu_ce_error() in g1
	rdpr	%pil, %g6		! read pil into %g6
	subcc	%g6, PIL_15, %g0
	  movneg	%icc, PIL_14, %g4 ! run at pil 14 unless already at 15
	sethi	%hi(sys_trap), %g5
	jmp	%g5 + %lo(sys_trap)	! goto sys_trap
	  movge	%icc, PIL_15, %g4	! already at pil 15
	SET_SIZE(ce_err)

	ENTRY_NP(ce_err_tl1)
#ifndef	TRAPTRACE
	ldxa	[%g0]ASI_AFSR, %g7
	stxa	%g7, [%g0]ASI_AFSR
	membar	#Sync
	retry
#else
	set	ce_trap_tl1, %g1
	sethi	%hi(dis_err_panic1), %g4
	jmp	%g4 + %lo(dis_err_panic1)
	nop
#endif
	SET_SIZE(ce_err_tl1)

#ifdef	TRAPTRACE
.celevel1msg:
	.asciz	"Softerror with trap tracing at tl1: AFAR 0x%08x.%08x AFSR 0x%08x.%08x";

	ENTRY_NP(ce_trap_tl1)
	! upper 32 bits of AFSR already in o3
	mov	%o4, %o0		! save AFAR upper 32 bits
	mov	%o2, %o4		! lower 32 bits of AFSR
	mov	%o1, %o2		! lower 32 bits of AFAR
	mov	%o0, %o1		! upper 32 bits of AFAR
	set	.celevel1msg, %o0
	call	panic
	nop
	SET_SIZE(ce_trap_tl1)
#endif

	!
	! async_err is the assembly glue code to get us from the actual trap
	! into the CPU module's C error handler.  Note that we also branch
	! here from ce_err() above.
	!
	ENTRY_NP(async_err)
	stxa	%g0, [%g0]ASI_ESTATE_ERR ! disable ecc and other cpu errors
	membar	#Sync			! membar sync required

	ldxa	[%g0]ASI_AFSR, %g3	! save afsr in g3
	ldxa	[%g0]ASI_AFAR, %g2	! save afar in g2

	sllx	%g5, 53, %g5		! move ttype to <63:53>
	or	%g3, %g5, %g3		! or to afsr in g3

	or	%g0, 1, %g1		! put 1 in g1
	sllx	%g1, 21, %g1		! shift left to <21> afsr UE
	andcc	%g1, %g3, %g0		! check for UE in afsr
	bz,a,pn %icc, 2f		! if !UE skip sdb read/clear
	  nop

	set	P_DER_H, %g4		! put P_DER_H in g4
	ldxa	[%g4]ASI_SDB_INTR_R, %g5 ! read sdb upper half into 56
	or	%g0, 1, %g6		! put 1 in g6
	sllx	%g6, 9, %g6		! shift g6 to <9> sdb UE
	andcc	%g5, %g6, %g1		! check for UE in upper half
	sllx	%g5, 33, %g5		! shift upper bits to <42:33>
	or	%g3, %g5, %g3		! or with afsr bits
	bz,a	1f			! no error, goto 1f
	  nop
	stxa	%g1, [%g4]ASI_SDB_INTR_W ! clear sdb reg UE error bit
	membar	#Sync			! membar sync required
1:
	set	P_DER_L, %g4		! put P_DER_L in g4
	ldxa	[%g4]ASI_SDB_INTR_R, %g5 ! read sdb lower half into g5
	andcc	%g5, %g6, %g1		! check for UE in lower half
	sllx	%g5, 43, %g5		! shift upper bits to <52:43>
	or	%g3, %g5, %g3		! or with afsr bits
	bz,a	2f			! no error, goto 2f
	  nop
	stxa	%g1, [%g4]ASI_SDB_INTR_W ! clear sdb reg UE error bit
	membar	#Sync			! membar sync required
2:
	stxa	%g3, [%g0]ASI_AFSR	! clear all the sticky bits
	membar	#Sync			! membar sync required

	RESET_USER_RTT_REGS(%g4, %g5, async_err_resetskip)
async_err_resetskip:

	set	cpu_async_error, %g1	! put cpu_async_error in g1
	sethi	%hi(sys_trap), %g5
	jmp	%g5 + %lo(sys_trap)	! goto sys_trap
	  or	%g0, PIL_15, %g4	! run at pil 15
	SET_SIZE(async_err)

	ENTRY_NP(dis_err_panic1)
	stxa	%g0, [%g0]ASI_ESTATE_ERR ! disable all error traps
	membar	#Sync
	! save destination routine is in g1
	ldxa	[%g0]ASI_AFAR, %g2	! read afar
	ldxa	[%g0]ASI_AFSR, %g3	! read afsr
	set	P_DER_H, %g4		! put P_DER_H in g4
	ldxa	[%g4]ASI_SDB_INTR_R, %g5 ! read sdb upper half into g5
	sllx	%g5, 33, %g5		! shift upper bits to <42:33>
	or	%g3, %g5, %g3		! or with afsr bits
	set	P_DER_L, %g4		! put P_DER_L in g4
	ldxa	[%g4]ASI_SDB_INTR_R, %g5 ! read sdb lower half into g5
	sllx	%g5, 43, %g5		! shift upper bits to <52:43>
	or	%g3, %g5, %g3		! or with afsr bits

	RESET_USER_RTT_REGS(%g4, %g5, dis_err_panic1_resetskip)
dis_err_panic1_resetskip:

	sethi	%hi(sys_trap), %g5
	jmp	%g5 + %lo(sys_trap)	! goto sys_trap
	  sub	%g0, 1, %g4
	SET_SIZE(dis_err_panic1)

	ENTRY(clr_datapath)
	set	P_DER_H, %o4			! put P_DER_H in o4
	ldxa	[%o4]ASI_SDB_INTR_R, %o5	! read sdb upper half into o3
	or	%g0, 0x3, %o2			! put 0x3 in o2
	sllx	%o2, 8, %o2			! shift o2 to <9:8> sdb
	andcc	%o5, %o2, %o1			! check for UE,CE in upper half
	bz,a	1f				! no error, goto 1f
	  nop
	stxa	%o1, [%o4]ASI_SDB_INTR_W	! clear sdb reg UE,CE error bits
	membar	#Sync				! membar sync required
1:
	set	P_DER_L, %o4			! put P_DER_L in o4
	ldxa	[%o4]ASI_SDB_INTR_R, %o5	! read sdb lower half into o5
	andcc	%o5, %o2, %o1			! check for UE,CE in lower half
	bz,a	2f				! no error, goto 2f
	  nop
	stxa	%o1, [%o4]ASI_SDB_INTR_W	! clear sdb reg UE,CE error bits
	membar	#Sync
2:
	retl
	  nop
	SET_SIZE(clr_datapath)

	ENTRY(get_udb_errors)
	set	P_DER_H, %o3
	ldxa	[%o3]ASI_SDB_INTR_R, %o2
	stx	%o2, [%o0]
	set	P_DER_L, %o3
	ldxa	[%o3]ASI_SDB_INTR_R, %o2
	retl
	  stx	%o2, [%o1]
	SET_SIZE(get_udb_errors)

#endif /* lint */

#if defined(lint)
/*
 * The itlb_rd_entry and dtlb_rd_entry functions return the tag portion of the
 * tte, the virtual address, and the ctxnum of the specified tlb entry.  They
 * should only be used in places where you have no choice but to look at the
 * tlb itself.
 *
 * Note: These two routines are required by the Estar "cpr" loadable module.
 */
/*ARGSUSED*/
void
itlb_rd_entry(uint_t entry, tte_t *tte, uint64_t *va_tag)
{}

/*ARGSUSED*/
void
dtlb_rd_entry(uint_t entry, tte_t *tte, uint64_t *va_tag)
{}
#else 	/* lint */
/*
 * NB - In Spitfire cpus, when reading a tte from the hardware, we
 * need to clear [42-41] because the general definitions in pte.h
 * define the PA to be [42-13] whereas Spitfire really uses [40-13].
 * When cloning these routines for other cpus the "andn" below is not
 * necessary.
 */
	ENTRY_NP(itlb_rd_entry)
	sllx	%o0, 3, %o0
#if defined(SF_ERRATA_32)
	sethi	%hi(FLUSH_ADDR), %g2
	set	MMU_PCONTEXT, %g1
	stxa	%g0, [%g1]ASI_DMMU			! KCONTEXT
	flush	%g2
#endif
	ldxa	[%o0]ASI_ITLB_ACCESS, %g1
	set	TTE_SPITFIRE_PFNHI_CLEAR, %g2		! spitfire only
	sllx	%g2, TTE_SPITFIRE_PFNHI_SHIFT, %g2	! see comment above
	andn	%g1, %g2, %g1				! for details
	stx	%g1, [%o1]
	ldxa	[%o0]ASI_ITLB_TAGREAD, %g2
	set	TAGREAD_CTX_MASK, %o4
	andn	%g2, %o4, %o5
	retl
	  stx	%o5, [%o2]
	SET_SIZE(itlb_rd_entry)

	ENTRY_NP(dtlb_rd_entry)
	sllx	%o0, 3, %o0
#if defined(SF_ERRATA_32)
	sethi	%hi(FLUSH_ADDR), %g2
	set	MMU_PCONTEXT, %g1
	stxa	%g0, [%g1]ASI_DMMU			! KCONTEXT
	flush	%g2
#endif
	ldxa	[%o0]ASI_DTLB_ACCESS, %g1
	set	TTE_SPITFIRE_PFNHI_CLEAR, %g2		! spitfire only
	sllx	%g2, TTE_SPITFIRE_PFNHI_SHIFT, %g2	! see comment above
	andn	%g1, %g2, %g1				! itlb_rd_entry
	stx	%g1, [%o1]
	ldxa	[%o0]ASI_DTLB_TAGREAD, %g2
	set	TAGREAD_CTX_MASK, %o4
	andn	%g2, %o4, %o5
	retl
	  stx	%o5, [%o2]
	SET_SIZE(dtlb_rd_entry)
#endif /* lint */

#if defined(lint)

/*
 * routines to get and set the LSU register
 */
uint64_t
get_lsu(void)
{
	return ((uint64_t)0);
}

/*ARGSUSED*/
void
set_lsu(uint64_t lsu)
{}

#else /* lint */

	ENTRY(set_lsu)
	stxa	%o0, [%g0]ASI_LSU		! store to LSU
	retl
	membar	#Sync
	SET_SIZE(set_lsu)

	ENTRY(get_lsu)
	retl
	ldxa	[%g0]ASI_LSU, %o0		! load LSU
	SET_SIZE(get_lsu)

#endif /* lint */

#ifndef lint
	/*
	 * Clear the NPT (non-privileged trap) bit in the %tick
	 * registers. In an effort to make the change in the
	 * tick counter as consistent as possible, we disable
	 * all interrupts while we're changing the registers. We also
	 * ensure that the read and write instructions are in the same
	 * line in the instruction cache.
	 */
	ENTRY_NP(cpu_clearticknpt)
	rdpr	%pstate, %g1		/* save processor state */
	andn	%g1, PSTATE_IE, %g3	/* turn off */
	wrpr	%g0, %g3, %pstate	/*   interrupts */
	rdpr	%tick, %g2		/* get tick register */
	brgez,pn %g2, 1f		/* if NPT bit off, we're done */
	mov	1, %g3			/* create mask */
	sllx	%g3, 63, %g3		/*   for NPT bit */
	ba,a,pt	%xcc, 2f
	.align	64			/* Align to I$ boundary */
2:
	rdpr	%tick, %g2		/* get tick register */
	wrpr	%g3, %g2, %tick		/* write tick register, */
					/*   clearing NPT bit   */
#if defined(BB_ERRATA_1)
	rdpr	%tick, %g0		/* read (s)tick (BB_ERRATA_1) */
#endif
1:
	jmp	%g4 + 4
	wrpr	%g0, %g1, %pstate	/* restore processor state */
	SET_SIZE(cpu_clearticknpt)

	/*
	 * get_ecache_tag()
	 * Register Usage:
	 * %o0: In: 32-bit E$ index
	 *      Out: 64-bit E$ tag value
	 * %o1: In: 64-bit AFSR value after clearing sticky bits
	 * %o2: In: address of cpu private afsr storage
	 */
	ENTRY(get_ecache_tag)
	or	%g0, 1, %o4
	sllx	%o4, 40, %o4			! set bit 40 for e$ tag access
	or	%o0, %o4, %o4			! %o4 = e$ addr for tag read
	rdpr	%pstate, %o5
	andn	%o5, PSTATE_IE | PSTATE_AM, %o0
	wrpr	%o0, %g0, %pstate		! clear IE, AM bits

	ldxa	[%g0]ASI_ESTATE_ERR, %g1
	stxa	%g0, [%g0]ASI_ESTATE_ERR	! Turn off Error enable
	membar	#Sync

	ldxa	[%g0]ASI_AFSR, %o0
	srlx	%o0, P_AFSR_CP_SHIFT, %o3
	btst	1, %o3
	bz	1f
	  nop
	ldx	[%o2], %g4
	or	%g4, %o0, %g4			! aggregate AFSR in cpu private
	stx	%g4, [%o2]
1:
	stxa	%o0, [%g0]ASI_AFSR		! clear AFSR
	membar  #Sync

	ldxa	[%o4]ASI_EC_R, %g0
	ldxa	[%g0]ASI_EC_DIAG, %o0		! read tag from e$ tag reg

	ldxa	[%g0]ASI_AFSR, %o3
	srlx	%o3, P_AFSR_CP_SHIFT, %o4
	btst	1, %o4
	bz	2f
	  stx	%o3, [%o1]			! AFSR after sticky clear
	ldx	[%o2], %g4
	or	%g4, %o3, %g4			! aggregate AFSR in cpu private
	stx	%g4, [%o2]
2:
	membar	#Sync

	stxa	%g1, [%g0]ASI_ESTATE_ERR	! Turn error enable back on
	membar	#Sync
	retl
	wrpr	%g0, %o5, %pstate
	SET_SIZE(get_ecache_tag)

	/*
	 * check_ecache_line()
	 * Register Usage:
	 * %o0: In: 32-bit E$ index
	 *      Out: 64-bit accumulated AFSR
	 * %o1: In: address of cpu private afsr storage
	 */
	ENTRY(check_ecache_line)
	or	%g0, 1, %o4
	sllx	%o4, 39, %o4			! set bit 39 for e$ data access
	or	%o0, %o4, %o4		 	! %o4 = e$ addr for data read

	rdpr	%pstate, %o5
	andn	%o5, PSTATE_IE | PSTATE_AM, %o0
	wrpr	%o0, %g0, %pstate		! clear IE, AM bits

	ldxa	[%g0]ASI_ESTATE_ERR, %g1
	stxa	%g0, [%g0]ASI_ESTATE_ERR 	! Turn off Error enable
	membar	#Sync

	ldxa 	[%g0]ASI_AFSR, %o0
	srlx	%o0, P_AFSR_CP_SHIFT, %o2
	btst	1, %o2
	bz	1f
	  clr	%o2				! loop count
	ldx	[%o1], %o3
	or	%o3, %o0, %o3			! aggregate AFSR in cpu private
	stx	%o3, [%o1]
1:
	stxa    %o0, [%g0]ASI_AFSR              ! clear AFSR
	membar	#Sync

2:
	ldxa	[%o4]ASI_EC_R, %g0		! Read the E$ data 8bytes each
	add	%o2, 1, %o2
	cmp	%o2, 8
	bl,a 	2b
	  add	%o4, 8, %o4

	membar	#Sync
	ldxa	[%g0]ASI_AFSR, %o0		! read accumulated AFSR
	srlx	%o0, P_AFSR_CP_SHIFT, %o2
	btst	1, %o2
	bz	3f
	  nop
	ldx	[%o1], %o3
	or	%o3, %o0, %o3			! aggregate AFSR in cpu private
	stx	%o3, [%o1]
3:
	stxa	%o0, [%g0]ASI_AFSR		! clear AFSR
	membar	#Sync
	stxa	%g1, [%g0]ASI_ESTATE_ERR	! Turn error enable back on
	membar	#Sync
	retl
	wrpr	%g0, %o5, %pstate
	SET_SIZE(check_ecache_line)
#endif /* lint */

#if defined(lint)
uint64_t
read_and_clear_afsr()
{
	return ((uint64_t)0);
}
#else	/* lint */
	ENTRY(read_and_clear_afsr)
	ldxa	[%g0]ASI_AFSR, %o0
	retl
	  stxa	%o0, [%g0]ASI_AFSR		! clear AFSR
	SET_SIZE(read_and_clear_afsr)
#endif	/* lint */

#if defined(lint)
/* ARGSUSED */
void
scrubphys(uint64_t paddr, int ecache_size)
{
}

#else	/* lint */

/*
 * scrubphys - Pass in the aligned physical memory address that you want
 * to scrub, along with the ecache size.
 *
 *	1) Displacement flush the E$ line corresponding to %addr.
 *	   The first ldxa guarantees that the %addr is no longer in
 *	   M, O, or E (goes to I or S (if instruction fetch also happens).
 *	2) "Write" the data using a CAS %addr,%g0,%g0.
 *	   The casxa guarantees a transition from I to M or S to M.
 *	3) Displacement flush the E$ line corresponding to %addr.
 *	   The second ldxa pushes the M line out of the ecache, into the
 *	   writeback buffers, on the way to memory.
 *	4) The "membar #Sync" pushes the cache line out of the writeback
 *	   buffers onto the bus, on the way to dram finally.
 *
 * This is a modified version of the algorithm suggested by Gary Lauterbach.
 * In theory the CAS %addr,%g0,%g0 is supposed to mark the addr's cache line
 * as modified, but then we found out that for spitfire, if it misses in the
 * E$ it will probably install as an M, but if it hits in the E$, then it
 * will stay E, if the store doesn't happen. So the first displacement flush
 * should ensure that the CAS will miss in the E$.  Arrgh.
 */

	ENTRY(scrubphys)
	or	%o1, %g0, %o2	! put ecache size in %o2
#ifndef HUMMINGBIRD
	xor	%o0, %o2, %o1	! calculate alias address
	add	%o2, %o2, %o3	! 2 * ecachesize in case
				! addr == ecache_flushaddr
	sub	%o3, 1, %o3	! -1 == mask
	and	%o1, %o3, %o1	! and with xor'd address
	set	ecache_flushaddr, %o3
	ldx	[%o3], %o3

	rdpr	%pstate, %o4
	andn	%o4, PSTATE_IE | PSTATE_AM, %o5
	wrpr	%o5, %g0, %pstate	! clear IE, AM bits

	ldxa	[%o1 + %o3]ASI_MEM, %g0 ! load ecache_flushaddr + alias
	casxa	[%o0]ASI_MEM, %g0, %g0
	ldxa	[%o1 + %o3]ASI_MEM, %g0	! load ecache_flushaddr + alias

#else /* HUMMINGBIRD */
	/*
	 * UltraSPARC-IIe processor supports both 4-way set associative
	 * and direct map E$. We need to reconfigure E$ to direct map
	 * mode for data load/store before displacement flush. Also, we
	 * need to flush all 4 sets of the E$ to ensure that the physaddr
	 * has been flushed. Keep the interrupts disabled while flushing
	 * E$ in this manner.
	 *
	 * For flushing a specific physical address, we start at the
	 * aliased address and load at set-size stride, wrapping around
	 * at 2*ecache-size boundary and skipping fault physical address.
	 * It takes 10 loads to guarantee that the physical address has
	 * been flushed.
	 *
	 * Usage:
	 *	%o0	physaddr
	 *	%o5	physaddr - ecache_flushaddr
	 *	%g1	UPA config (restored later)
	 *	%g2	E$ set size
	 *	%g3	E$ flush address range mask (i.e. 2 * E$ -1)
	 *	%g4	#loads to flush phys address
	 *	%g5	temp
	 */

	sethi	%hi(ecache_associativity), %g5
	ld	[%g5 + %lo(ecache_associativity)], %g5
	udivx	%o2, %g5, %g2	! set size (i.e. ecache_size/#sets)
	xor	%o0, %o2, %o1	! calculate alias address
	add	%o2, %o2, %g3	! 2 * ecachesize in case
				! addr == ecache_flushaddr
	sub	%g3, 1, %g3	! 2 * ecachesize -1 == mask
	and	%o1, %g3, %o1	! and with xor'd address
	sethi	%hi(ecache_flushaddr), %o3
	ldx	[%o3 + %lo(ecache_flushaddr)], %o3

	rdpr	%pstate, %o4
	andn	%o4, PSTATE_IE | PSTATE_AM, %o5
	wrpr	%o5, %g0, %pstate	! clear IE, AM bits

	! Place E$ in direct map mode for data access
	or	%g0, 1, %g5
	sllx	%g5, HB_UPA_DMAP_DATA_BIT, %g5
	ldxa	[%g0]ASI_UPA_CONFIG, %g1 ! current UPA config (restored later)
	or	%g1, %g5, %g5
	membar	#Sync
	stxa	%g5, [%g0]ASI_UPA_CONFIG ! enable direct map for data access
	membar	#Sync

	! Displace cache line from each set of E$ starting at the
	! aliased address. at set-size stride, wrapping at 2*ecache_size
	! and skipping load from physaddr. We need 10 loads to flush the
	! physaddr from E$.
	mov	HB_PHYS_FLUSH_CNT-1, %g4 ! #loads to flush phys addr
	sub	%o0, %o3, %o5		! physaddr - ecache_flushaddr
	or	%o1, %g0, %g5		! starting aliased offset
2:
	ldxa	[%g5 + %o3]ASI_MEM, %g0	! load ecache_flushaddr + alias
1:
	add	%g5, %g2, %g5		! calculate offset in next set
	and	%g5, %g3, %g5		! force offset within aliased range
	cmp	%g5, %o5		! skip loads from physaddr
	be,pn %ncc, 1b
	  nop
	brgz,pt	%g4, 2b
	  dec	%g4

	casxa	[%o0]ASI_MEM, %g0, %g0

	! Flush %o0 from ecahe again.
	! Need single displacement flush at offset %o1 this time as
	! the E$ is already in direct map mode.
	ldxa	[%o1 + %o3]ASI_MEM, %g0	! load ecache_flushaddr + alias

	membar	#Sync
	stxa	%g1, [%g0]ASI_UPA_CONFIG ! restore UPA config (DM bits)
	membar	#Sync
#endif /* HUMMINGBIRD */
	wrpr	%g0, %o4, %pstate	! restore earlier pstate register value

	retl
	membar	#Sync			! move the data out of the load buffer
	SET_SIZE(scrubphys)

#endif	/* lint */

#if defined(lint)

/*
 * clearphys - Pass in the aligned physical memory address that you want
 * to push out, as a 64 byte block of zeros, from the ecache zero-filled.
 * Since this routine does not bypass the ecache, it is possible that
 * it could generate a UE error while trying to clear the a bad line.
 * This routine clears and restores the error enable flag.
 * TBD - Hummingbird may need similar protection
 */
/* ARGSUSED */
void
clearphys(uint64_t paddr, int ecache_size, int ecache_linesize)
{
}

#else	/* lint */

	ENTRY(clearphys)
	or	%o2, %g0, %o3	! ecache linesize
	or	%o1, %g0, %o2	! ecache size
#ifndef HUMMINGBIRD
	or	%o3, %g0, %o4	! save ecache linesize
	xor	%o0, %o2, %o1	! calculate alias address
	add	%o2, %o2, %o3	! 2 * ecachesize
	sub	%o3, 1, %o3	! -1 == mask
	and	%o1, %o3, %o1	! and with xor'd address
	set	ecache_flushaddr, %o3
	ldx	[%o3], %o3
	or	%o4, %g0, %o2	! saved ecache linesize

	rdpr	%pstate, %o4
	andn	%o4, PSTATE_IE | PSTATE_AM, %o5
	wrpr	%o5, %g0, %pstate	! clear IE, AM bits

	ldxa	[%g0]ASI_ESTATE_ERR, %g1
	stxa	%g0, [%g0]ASI_ESTATE_ERR	! disable errors
	membar	#Sync

	! need to put zeros in the cache line before displacing it

	sub	%o2, 8, %o2	! get offset of last double word in ecache line
1:
	stxa	%g0, [%o0 + %o2]ASI_MEM	! put zeros in the ecache line
	sub	%o2, 8, %o2
	brgez,a,pt %o2, 1b
	nop
	ldxa	[%o1 + %o3]ASI_MEM, %g0	! load ecache_flushaddr + alias
	casxa	[%o0]ASI_MEM, %g0, %g0
	ldxa	[%o1 + %o3]ASI_MEM, %g0	! load ecache_flushaddr + alias

	stxa	%g1, [%g0]ASI_ESTATE_ERR	! restore error enable
	membar	#Sync

#else /* HUMMINGBIRD... */
	/*
	 * UltraSPARC-IIe processor supports both 4-way set associative
	 * and direct map E$. We need to reconfigure E$ to direct map
	 * mode for data load/store before displacement flush. Also, we
	 * need to flush all 4 sets of the E$ to ensure that the physaddr
	 * has been flushed. Keep the interrupts disabled while flushing
	 * E$ in this manner.
	 *
	 * For flushing a specific physical address, we start at the
	 * aliased address and load at set-size stride, wrapping around
	 * at 2*ecache-size boundary and skipping fault physical address.
	 * It takes 10 loads to guarantee that the physical address has
	 * been flushed.
	 *
	 * Usage:
	 *	%o0	physaddr
	 *	%o5	physaddr - ecache_flushaddr
	 *	%g1	UPA config (restored later)
	 *	%g2	E$ set size
	 *	%g3	E$ flush address range mask (i.e. 2 * E$ -1)
	 *	%g4	#loads to flush phys address
	 *	%g5	temp
	 */

	or	%o3, %g0, %o4	! save ecache linesize
	sethi	%hi(ecache_associativity), %g5
	ld	[%g5 + %lo(ecache_associativity)], %g5
	udivx	%o2, %g5, %g2	! set size (i.e. ecache_size/#sets)

	xor	%o0, %o2, %o1	! calculate alias address
	add	%o2, %o2, %g3	! 2 * ecachesize
	sub	%g3, 1, %g3	! 2 * ecachesize -1 == mask
	and	%o1, %g3, %o1	! and with xor'd address
	sethi	%hi(ecache_flushaddr), %o3
	ldx	[%o3 +%lo(ecache_flushaddr)], %o3
	or	%o4, %g0, %o2	! saved ecache linesize

	rdpr	%pstate, %o4
	andn	%o4, PSTATE_IE | PSTATE_AM, %o5
	wrpr	%o5, %g0, %pstate	! clear IE, AM bits

	! Place E$ in direct map mode for data access
	or	%g0, 1, %g5
	sllx	%g5, HB_UPA_DMAP_DATA_BIT, %g5
	ldxa	[%g0]ASI_UPA_CONFIG, %g1 ! current UPA config (restored later)
	or	%g1, %g5, %g5
	membar	#Sync
	stxa	%g5, [%g0]ASI_UPA_CONFIG ! enable direct map for data access
	membar	#Sync

	! need to put zeros in the cache line before displacing it

	sub	%o2, 8, %o2	! get offset of last double word in ecache line
1:
	stxa	%g0, [%o0 + %o2]ASI_MEM	! put zeros in the ecache line
	sub	%o2, 8, %o2
	brgez,a,pt %o2, 1b
	nop

	! Displace cache line from each set of E$ starting at the
	! aliased address. at set-size stride, wrapping at 2*ecache_size
	! and skipping load from physaddr. We need 10 loads to flush the
	! physaddr from E$.
	mov	HB_PHYS_FLUSH_CNT-1, %g4 ! #loads to flush phys addr
	sub	%o0, %o3, %o5		! physaddr - ecache_flushaddr
	or	%o1, %g0, %g5		! starting offset
2:
	ldxa	[%g5 + %o3]ASI_MEM, %g0	! load ecache_flushaddr + alias
3:
	add	%g5, %g2, %g5		! calculate offset in next set
	and	%g5, %g3, %g5		! force offset within aliased range
	cmp	%g5, %o5		! skip loads from physaddr
	be,pn %ncc, 3b
	  nop
	brgz,pt	%g4, 2b
	  dec	%g4

	casxa	[%o0]ASI_MEM, %g0, %g0

	! Flush %o0 from ecahe again.
	! Need single displacement flush at offset %o1 this time as
	! the E$ is already in direct map mode.
	ldxa	[%o1 + %o3]ASI_MEM, %g0	! load ecache_flushaddr + alias

	membar	#Sync
	stxa	%g1, [%g0]ASI_UPA_CONFIG ! restore UPA config (DM bits)
	membar	#Sync
#endif /* HUMMINGBIRD... */

	retl
	wrpr	%g0, %o4, %pstate	! restore earlier pstate register value
	SET_SIZE(clearphys)

#endif	/* lint */

#if defined(lint)
/* ARGSUSED */
void
flushecacheline(uint64_t paddr, int ecache_size)
{
}

#else	/* lint */
/*
 * flushecacheline - This is a simpler version of scrubphys
 * which simply does a displacement flush of the line in
 * question. This routine is mainly used in handling async
 * errors where we want to get rid of a bad line in ecache.
 * Note that if the line is modified and it has suffered
 * data corruption - we are guarantee that the hw will write
 * a UE back to mark the page poisoned.
 */
        ENTRY(flushecacheline)
        or      %o1, %g0, %o2   ! put ecache size in %o2
#ifndef HUMMINGBIRD
        xor     %o0, %o2, %o1   ! calculate alias address
        add     %o2, %o2, %o3   ! 2 * ecachesize in case
                                ! addr == ecache_flushaddr
        sub     %o3, 1, %o3     ! -1 == mask
        and     %o1, %o3, %o1   ! and with xor'd address
        set     ecache_flushaddr, %o3
        ldx     [%o3], %o3

        rdpr    %pstate, %o4
        andn    %o4, PSTATE_IE | PSTATE_AM, %o5
        wrpr    %o5, %g0, %pstate       ! clear IE, AM bits

	ldxa	[%g0]ASI_ESTATE_ERR, %g1
	stxa	%g0, [%g0]ASI_ESTATE_ERR	! disable errors
	membar	#Sync

        ldxa    [%o1 + %o3]ASI_MEM, %g0 ! load ecache_flushaddr + alias
	membar	#Sync
	stxa	%g1, [%g0]ASI_ESTATE_ERR	! restore error enable
        membar  #Sync
#else /* HUMMINGBIRD */
	/*
	 * UltraSPARC-IIe processor supports both 4-way set associative
	 * and direct map E$. We need to reconfigure E$ to direct map
	 * mode for data load/store before displacement flush. Also, we
	 * need to flush all 4 sets of the E$ to ensure that the physaddr
	 * has been flushed. Keep the interrupts disabled while flushing
	 * E$ in this manner.
	 *
	 * For flushing a specific physical address, we start at the
	 * aliased address and load at set-size stride, wrapping around
	 * at 2*ecache-size boundary and skipping fault physical address.
	 * It takes 10 loads to guarantee that the physical address has
	 * been flushed.
	 *
	 * Usage:
	 *	%o0	physaddr
	 *	%o5	physaddr - ecache_flushaddr
	 *	%g1	error enable register
	 *	%g2	E$ set size
	 *	%g3	E$ flush address range mask (i.e. 2 * E$ -1)
	 *	%g4	UPA config (restored later)
	 *	%g5	temp
	 */

	sethi	%hi(ecache_associativity), %g5
	ld	[%g5 + %lo(ecache_associativity)], %g5
	udivx	%o2, %g5, %g2	! set size (i.e. ecache_size/#sets)
	xor	%o0, %o2, %o1	! calculate alias address
	add	%o2, %o2, %g3	! 2 * ecachesize in case
				! addr == ecache_flushaddr
	sub	%g3, 1, %g3	! 2 * ecachesize -1 == mask
	and	%o1, %g3, %o1	! and with xor'd address
	sethi	%hi(ecache_flushaddr), %o3
	ldx	[%o3 + %lo(ecache_flushaddr)], %o3

	rdpr	%pstate, %o4
	andn	%o4, PSTATE_IE | PSTATE_AM, %o5
	wrpr	%o5, %g0, %pstate	! clear IE, AM bits

	! Place E$ in direct map mode for data access
	or	%g0, 1, %g5
	sllx	%g5, HB_UPA_DMAP_DATA_BIT, %g5
	ldxa	[%g0]ASI_UPA_CONFIG, %g4 ! current UPA config (restored later)
	or	%g4, %g5, %g5
	membar	#Sync
	stxa	%g5, [%g0]ASI_UPA_CONFIG ! enable direct map for data access
	membar	#Sync

	ldxa	[%g0]ASI_ESTATE_ERR, %g1
	stxa	%g0, [%g0]ASI_ESTATE_ERR	! disable errors
	membar	#Sync

	! Displace cache line from each set of E$ starting at the
	! aliased address. at set-size stride, wrapping at 2*ecache_size
	! and skipping load from physaddr. We need 10 loads to flush the
	! physaddr from E$.
	mov	HB_PHYS_FLUSH_CNT-1, %g5 ! #loads to flush physaddr
	sub	%o0, %o3, %o5		! physaddr - ecache_flushaddr
2:
	ldxa	[%o1 + %o3]ASI_MEM, %g0	! load ecache_flushaddr + alias
3:
	add	%o1, %g2, %o1		! calculate offset in next set
	and	%o1, %g3, %o1		! force offset within aliased range
	cmp	%o1, %o5		! skip loads from physaddr
	be,pn %ncc, 3b
	  nop
	brgz,pt	%g5, 2b
	  dec	%g5

	membar	#Sync
	stxa	%g1, [%g0]ASI_ESTATE_ERR	! restore error enable
        membar  #Sync

	stxa	%g4, [%g0]ASI_UPA_CONFIG ! restore UPA config (DM bits)
	membar	#Sync
#endif /* HUMMINGBIRD */
        retl
        wrpr    %g0, %o4, %pstate
        SET_SIZE(flushecacheline)

#endif	/* lint */

#if defined(lint)
/* ARGSUSED */
void
ecache_scrubreq_tl1(uint64_t inum, uint64_t dummy)
{
}

#else	/* lint */
/*
 * ecache_scrubreq_tl1 is the crosstrap handler called at ecache_calls_a_sec Hz
 * from the clock CPU.  It atomically increments the outstanding request
 * counter and, if there was not already an outstanding request,
 * branches to setsoftint_tl1 to enqueue an intr_vec for the given inum.
 */

	! Register usage:
	!
	! Arguments:
	! %g1 - inum
	!
	! Internal:
	! %g2, %g3, %g5 - scratch
	! %g4 - ptr. to spitfire_scrub_misc ec_scrub_outstanding.
	! %g6 - setsoftint_tl1 address

	ENTRY_NP(ecache_scrubreq_tl1)
	set	SFPR_SCRUB_MISC + EC_SCRUB_OUTSTANDING, %g2
	GET_CPU_PRIVATE_PTR(%g2, %g4, %g5, 1f);
	ld	[%g4], %g2		! cpu's ec_scrub_outstanding.
	set	setsoftint_tl1, %g6
	!
	! no need to use atomic instructions for the following
	! increment - we're at tl1
	!
	add	%g2, 0x1, %g3
	brnz,pn	%g2, 1f			! no need to enqueue more intr_vec
	  st	%g3, [%g4]		! delay - store incremented counter
	jmp	%g6			! setsoftint_tl1(%g1) - queue intr_vec
	  nop
	! not reached
1:
	retry
	SET_SIZE(ecache_scrubreq_tl1)

#endif	/* lint */

#if defined(lint)
/*ARGSUSED*/
void
write_ec_tag_parity(uint32_t id)
{}
#else /* lint */

	/*
         * write_ec_tag_parity(), which zero's the ecache tag,
         * marks the state as invalid and writes good parity to the tag.
         * Input %o1= 32 bit E$ index
         */
        ENTRY(write_ec_tag_parity)
        or      %g0, 1, %o4
        sllx    %o4, 39, %o4                    ! set bit 40 for e$ tag access
        or      %o0, %o4, %o4                 ! %o4 = ecache addr for tag write

        rdpr    %pstate, %o5
        andn    %o5, PSTATE_IE | PSTATE_AM, %o1
        wrpr    %o1, %g0, %pstate               ! clear IE, AM bits

        ldxa    [%g0]ASI_ESTATE_ERR, %g1
        stxa    %g0, [%g0]ASI_ESTATE_ERR        ! Turn off Error enable
        membar  #Sync

        ba      1f
         nop
	/*
         * Align on the ecache boundary in order to force
         * ciritical code section onto the same ecache line.
         */
         .align 64

1:
        set     S_EC_PARITY, %o3         	! clear tag, state invalid
        sllx    %o3, S_ECPAR_SHIFT, %o3   	! and with good tag parity
        stxa    %o3, [%g0]ASI_EC_DIAG           ! update with the above info
        stxa    %g0, [%o4]ASI_EC_W
        membar  #Sync

        stxa    %g1, [%g0]ASI_ESTATE_ERR        ! Turn error enable back on
        membar  #Sync
        retl
        wrpr    %g0, %o5, %pstate
        SET_SIZE(write_ec_tag_parity)

#endif /* lint */

#if defined(lint)
/*ARGSUSED*/
void
write_hb_ec_tag_parity(uint32_t id)
{}
#else /* lint */

	/*
         * write_hb_ec_tag_parity(), which zero's the ecache tag,
         * marks the state as invalid and writes good parity to the tag.
         * Input %o1= 32 bit E$ index
         */
        ENTRY(write_hb_ec_tag_parity)
        or      %g0, 1, %o4
        sllx    %o4, 39, %o4                    ! set bit 40 for e$ tag access
        or      %o0, %o4, %o4               ! %o4 = ecache addr for tag write

        rdpr    %pstate, %o5
        andn    %o5, PSTATE_IE | PSTATE_AM, %o1
        wrpr    %o1, %g0, %pstate               ! clear IE, AM bits

        ldxa    [%g0]ASI_ESTATE_ERR, %g1
        stxa    %g0, [%g0]ASI_ESTATE_ERR        ! Turn off Error enable
        membar  #Sync

        ba      1f
         nop
	/*
         * Align on the ecache boundary in order to force
         * ciritical code section onto the same ecache line.
         */
         .align 64
1:
#ifdef HUMMINGBIRD
        set     HB_EC_PARITY, %o3         	! clear tag, state invalid
        sllx    %o3, HB_ECPAR_SHIFT, %o3   	! and with good tag parity
#else /* !HUMMINGBIRD */
        set     SB_EC_PARITY, %o3         	! clear tag, state invalid
        sllx    %o3, SB_ECPAR_SHIFT, %o3   	! and with good tag parity
#endif /* !HUMMINGBIRD */

        stxa    %o3, [%g0]ASI_EC_DIAG           ! update with the above info
        stxa    %g0, [%o4]ASI_EC_W
        membar  #Sync

        stxa    %g1, [%g0]ASI_ESTATE_ERR        ! Turn error enable back on
        membar  #Sync
        retl
        wrpr    %g0, %o5, %pstate
        SET_SIZE(write_hb_ec_tag_parity)

#endif /* lint */

#define	VIS_BLOCKSIZE		64

#if defined(lint)

/*ARGSUSED*/
int
dtrace_blksuword32(uintptr_t addr, uint32_t *data, int tryagain)
{ return (0); }

#else

	ENTRY(dtrace_blksuword32)
	save	%sp, -SA(MINFRAME + 4), %sp

	rdpr	%pstate, %l1
	andn	%l1, PSTATE_IE, %l2		! disable interrupts to
	wrpr	%g0, %l2, %pstate		! protect our FPU diddling

	rd	%fprs, %l0
	andcc	%l0, FPRS_FEF, %g0
	bz,a,pt	%xcc, 1f			! if the fpu is disabled
	wr	%g0, FPRS_FEF, %fprs		! ... enable the fpu

	st	%f0, [%fp + STACK_BIAS - 4]	! save %f0 to the stack
1:
	set	0f, %l5
        /*
         * We're about to write a block full or either total garbage
         * (not kernel data, don't worry) or user floating-point data
         * (so it only _looks_ like garbage).
         */
	ld	[%i1], %f0			! modify the block
	membar	#Sync
	stn	%l5, [THREAD_REG + T_LOFAULT]	! set up the lofault handler
	stda	%d0, [%i0]ASI_BLK_COMMIT_S	! store the modified block
	membar	#Sync
	stn	%g0, [THREAD_REG + T_LOFAULT]	! remove the lofault handler

	bz,a,pt	%xcc, 1f
	wr	%g0, %l0, %fprs			! restore %fprs

	ld	[%fp + STACK_BIAS - 4], %f0	! restore %f0
1:

	wrpr	%g0, %l1, %pstate		! restore interrupts

	ret
	restore	%g0, %g0, %o0

0:
	membar	#Sync
	stn	%g0, [THREAD_REG + T_LOFAULT]	! remove the lofault handler

	bz,a,pt	%xcc, 1f
	wr	%g0, %l0, %fprs			! restore %fprs

	ld	[%fp + STACK_BIAS - 4], %f0	! restore %f0
1:

	wrpr	%g0, %l1, %pstate		! restore interrupts

	/*
	 * If tryagain is set (%i2) we tail-call dtrace_blksuword32_err()
	 * which deals with watchpoints. Otherwise, just return -1.
	 */
	brnz,pt	%i2, 1f
	nop
	ret
	restore	%g0, -1, %o0
1:
	call	dtrace_blksuword32_err
	restore

	SET_SIZE(dtrace_blksuword32)

#endif /* lint */