xref: /onnv-gate/usr/src/uts/i86pc/os/intr.c (revision 1455:b43f098fa50c)
10Sstevel@tonic-gate /*
20Sstevel@tonic-gate  * CDDL HEADER START
30Sstevel@tonic-gate  *
40Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5*1455Sandrei  * Common Development and Distribution License (the "License").
6*1455Sandrei  * You may not use this file except in compliance with the License.
70Sstevel@tonic-gate  *
80Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
90Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
100Sstevel@tonic-gate  * See the License for the specific language governing permissions
110Sstevel@tonic-gate  * and limitations under the License.
120Sstevel@tonic-gate  *
130Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
140Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
150Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
160Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
170Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
180Sstevel@tonic-gate  *
190Sstevel@tonic-gate  * CDDL HEADER END
200Sstevel@tonic-gate  */
210Sstevel@tonic-gate /*
22*1455Sandrei  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
230Sstevel@tonic-gate  * Use is subject to license terms.
240Sstevel@tonic-gate  */
250Sstevel@tonic-gate 
260Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
270Sstevel@tonic-gate 
280Sstevel@tonic-gate #include <sys/cpuvar.h>
290Sstevel@tonic-gate #include <sys/regset.h>
300Sstevel@tonic-gate #include <sys/psw.h>
310Sstevel@tonic-gate #include <sys/types.h>
320Sstevel@tonic-gate #include <sys/thread.h>
330Sstevel@tonic-gate #include <sys/systm.h>
340Sstevel@tonic-gate #include <sys/segments.h>
350Sstevel@tonic-gate #include <sys/pcb.h>
360Sstevel@tonic-gate #include <sys/trap.h>
370Sstevel@tonic-gate #include <sys/ftrace.h>
380Sstevel@tonic-gate #include <sys/traptrace.h>
390Sstevel@tonic-gate #include <sys/clock.h>
400Sstevel@tonic-gate #include <sys/panic.h>
410Sstevel@tonic-gate #include <sys/disp.h>
420Sstevel@tonic-gate #include <vm/seg_kp.h>
430Sstevel@tonic-gate #include <sys/stack.h>
440Sstevel@tonic-gate #include <sys/sysmacros.h>
450Sstevel@tonic-gate #include <sys/cmn_err.h>
460Sstevel@tonic-gate #include <sys/kstat.h>
470Sstevel@tonic-gate #include <sys/smp_impldefs.h>
480Sstevel@tonic-gate #include <sys/pool_pset.h>
490Sstevel@tonic-gate #include <sys/zone.h>
500Sstevel@tonic-gate #include <sys/bitmap.h>
510Sstevel@tonic-gate 
520Sstevel@tonic-gate #if defined(__amd64)
530Sstevel@tonic-gate 
540Sstevel@tonic-gate #if defined(__lint)
550Sstevel@tonic-gate /*
560Sstevel@tonic-gate  * atomic_btr32() is a gcc __inline__ function, defined in <asm/bitmap.h>
570Sstevel@tonic-gate  * For lint purposes, define it here.
580Sstevel@tonic-gate  */
590Sstevel@tonic-gate uint_t
600Sstevel@tonic-gate atomic_btr32(uint32_t *pending, uint_t pil)
610Sstevel@tonic-gate {
620Sstevel@tonic-gate 	return (*pending &= ~(1 << pil));
630Sstevel@tonic-gate }
640Sstevel@tonic-gate #else
650Sstevel@tonic-gate 
660Sstevel@tonic-gate extern uint_t atomic_btr32(uint32_t *pending, uint_t pil);
670Sstevel@tonic-gate 
680Sstevel@tonic-gate #endif
690Sstevel@tonic-gate 
700Sstevel@tonic-gate /*
710Sstevel@tonic-gate  * This code is amd64-only for now, but as time permits, we should
720Sstevel@tonic-gate  * use this on i386 too.
730Sstevel@tonic-gate  */
740Sstevel@tonic-gate 
750Sstevel@tonic-gate /*
760Sstevel@tonic-gate  * Some questions to ponder:
770Sstevel@tonic-gate  * -	in several of these routines, we make multiple calls to tsc_read()
780Sstevel@tonic-gate  *	without invoking functions .. couldn't we just reuse the same
790Sstevel@tonic-gate  *	timestamp sometimes?
800Sstevel@tonic-gate  * -	if we have the inline, we can probably make set_base_spl be a
810Sstevel@tonic-gate  *	C routine too.
820Sstevel@tonic-gate  */
830Sstevel@tonic-gate 
840Sstevel@tonic-gate static uint_t
850Sstevel@tonic-gate bsrw_insn(uint16_t mask)
860Sstevel@tonic-gate {
870Sstevel@tonic-gate 	uint_t index = sizeof (mask) * NBBY - 1;
880Sstevel@tonic-gate 
890Sstevel@tonic-gate 	ASSERT(mask != 0);
900Sstevel@tonic-gate 
910Sstevel@tonic-gate 	while ((mask & (1 << index)) == 0)
920Sstevel@tonic-gate 		index--;
930Sstevel@tonic-gate 	return (index);
940Sstevel@tonic-gate }
950Sstevel@tonic-gate 
960Sstevel@tonic-gate /*
970Sstevel@tonic-gate  * Do all the work necessary to set up the cpu and thread structures
980Sstevel@tonic-gate  * to dispatch a high-level interrupt.
990Sstevel@tonic-gate  *
1000Sstevel@tonic-gate  * Returns 0 if we're -not- already on the high-level interrupt stack,
1010Sstevel@tonic-gate  * (and *must* switch to it), non-zero if we are already on that stack.
1020Sstevel@tonic-gate  *
1030Sstevel@tonic-gate  * Called with interrupts masked.
1040Sstevel@tonic-gate  * The 'pil' is already set to the appropriate level for rp->r_trapno.
1050Sstevel@tonic-gate  */
1060Sstevel@tonic-gate int
1070Sstevel@tonic-gate hilevel_intr_prolog(struct cpu *cpu, uint_t pil, uint_t oldpil, struct regs *rp)
1080Sstevel@tonic-gate {
1090Sstevel@tonic-gate 	struct machcpu *mcpu = &cpu->cpu_m;
1100Sstevel@tonic-gate 	uint_t mask;
111590Sesolom 	hrtime_t intrtime;
1120Sstevel@tonic-gate 
1130Sstevel@tonic-gate 	ASSERT(pil > LOCK_LEVEL);
1140Sstevel@tonic-gate 
1150Sstevel@tonic-gate 	if (pil == CBE_HIGH_PIL) {
1160Sstevel@tonic-gate 		cpu->cpu_profile_pil = oldpil;
1170Sstevel@tonic-gate 		if (USERMODE(rp->r_cs)) {
1180Sstevel@tonic-gate 			cpu->cpu_profile_pc = 0;
1190Sstevel@tonic-gate 			cpu->cpu_profile_upc = rp->r_pc;
1200Sstevel@tonic-gate 		} else {
1210Sstevel@tonic-gate 			cpu->cpu_profile_pc = rp->r_pc;
1220Sstevel@tonic-gate 			cpu->cpu_profile_upc = 0;
1230Sstevel@tonic-gate 		}
1240Sstevel@tonic-gate 	}
1250Sstevel@tonic-gate 
1260Sstevel@tonic-gate 	mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK;
1270Sstevel@tonic-gate 	if (mask != 0) {
1280Sstevel@tonic-gate 		int nestpil;
1290Sstevel@tonic-gate 
1300Sstevel@tonic-gate 		/*
1310Sstevel@tonic-gate 		 * We have interrupted another high-level interrupt.
1320Sstevel@tonic-gate 		 * Load starting timestamp, compute interval, update
1330Sstevel@tonic-gate 		 * cumulative counter.
1340Sstevel@tonic-gate 		 */
1350Sstevel@tonic-gate 		nestpil = bsrw_insn((uint16_t)mask);
1360Sstevel@tonic-gate 		ASSERT(nestpil < pil);
137590Sesolom 		intrtime = tsc_read() -
1380Sstevel@tonic-gate 		    mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)];
139916Sschwartz 		mcpu->intrstat[nestpil][0] += intrtime;
140590Sesolom 		cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
1410Sstevel@tonic-gate 		/*
1420Sstevel@tonic-gate 		 * Another high-level interrupt is active below this one, so
1430Sstevel@tonic-gate 		 * there is no need to check for an interrupt thread.  That
1440Sstevel@tonic-gate 		 * will be done by the lowest priority high-level interrupt
1450Sstevel@tonic-gate 		 * active.
1460Sstevel@tonic-gate 		 */
1470Sstevel@tonic-gate 	} else {
1480Sstevel@tonic-gate 		kthread_t *t = cpu->cpu_thread;
1490Sstevel@tonic-gate 
1500Sstevel@tonic-gate 		/*
1510Sstevel@tonic-gate 		 * See if we are interrupting a low-level interrupt thread.
1520Sstevel@tonic-gate 		 * If so, account for its time slice only if its time stamp
1530Sstevel@tonic-gate 		 * is non-zero.
1540Sstevel@tonic-gate 		 */
1550Sstevel@tonic-gate 		if ((t->t_flag & T_INTR_THREAD) != 0 && t->t_intr_start != 0) {
156590Sesolom 			intrtime = tsc_read() - t->t_intr_start;
157916Sschwartz 			mcpu->intrstat[t->t_pil][0] += intrtime;
158590Sesolom 			cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
1590Sstevel@tonic-gate 			t->t_intr_start = 0;
1600Sstevel@tonic-gate 		}
1610Sstevel@tonic-gate 	}
1620Sstevel@tonic-gate 
1630Sstevel@tonic-gate 	/*
1640Sstevel@tonic-gate 	 * Store starting timestamp in CPU structure for this PIL.
1650Sstevel@tonic-gate 	 */
1660Sstevel@tonic-gate 	mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] = tsc_read();
1670Sstevel@tonic-gate 
1680Sstevel@tonic-gate 	ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
1690Sstevel@tonic-gate 
1700Sstevel@tonic-gate 	if (pil == 15) {
1710Sstevel@tonic-gate 		/*
1720Sstevel@tonic-gate 		 * To support reentrant level 15 interrupts, we maintain a
1730Sstevel@tonic-gate 		 * recursion count in the top half of cpu_intr_actv.  Only
1740Sstevel@tonic-gate 		 * when this count hits zero do we clear the PIL 15 bit from
1750Sstevel@tonic-gate 		 * the lower half of cpu_intr_actv.
1760Sstevel@tonic-gate 		 */
1770Sstevel@tonic-gate 		uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1;
1780Sstevel@tonic-gate 		(*refcntp)++;
1790Sstevel@tonic-gate 	}
1800Sstevel@tonic-gate 
1810Sstevel@tonic-gate 	mask = cpu->cpu_intr_actv;
1820Sstevel@tonic-gate 
1830Sstevel@tonic-gate 	cpu->cpu_intr_actv |= (1 << pil);
1840Sstevel@tonic-gate 
1850Sstevel@tonic-gate 	return (mask & CPU_INTR_ACTV_HIGH_LEVEL_MASK);
1860Sstevel@tonic-gate }
1870Sstevel@tonic-gate 
1880Sstevel@tonic-gate /*
1890Sstevel@tonic-gate  * Does most of the work of returning from a high level interrupt.
1900Sstevel@tonic-gate  *
1910Sstevel@tonic-gate  * Returns 0 if there are no more high level interrupts (in which
1920Sstevel@tonic-gate  * case we must switch back to the interrupted thread stack) or
1930Sstevel@tonic-gate  * non-zero if there are more (in which case we should stay on it).
1940Sstevel@tonic-gate  *
1950Sstevel@tonic-gate  * Called with interrupts masked
1960Sstevel@tonic-gate  */
1970Sstevel@tonic-gate int
1980Sstevel@tonic-gate hilevel_intr_epilog(struct cpu *cpu, uint_t pil, uint_t oldpil, uint_t vecnum)
1990Sstevel@tonic-gate {
2000Sstevel@tonic-gate 	struct machcpu *mcpu = &cpu->cpu_m;
2010Sstevel@tonic-gate 	uint_t mask;
202590Sesolom 	hrtime_t intrtime;
2030Sstevel@tonic-gate 
2040Sstevel@tonic-gate 	ASSERT(mcpu->mcpu_pri == pil);
2050Sstevel@tonic-gate 
2060Sstevel@tonic-gate 	cpu->cpu_stats.sys.intr[pil - 1]++;
2070Sstevel@tonic-gate 
2080Sstevel@tonic-gate 	ASSERT(cpu->cpu_intr_actv & (1 << pil));
2090Sstevel@tonic-gate 
2100Sstevel@tonic-gate 	if (pil == 15) {
2110Sstevel@tonic-gate 		/*
2120Sstevel@tonic-gate 		 * To support reentrant level 15 interrupts, we maintain a
2130Sstevel@tonic-gate 		 * recursion count in the top half of cpu_intr_actv.  Only
2140Sstevel@tonic-gate 		 * when this count hits zero do we clear the PIL 15 bit from
2150Sstevel@tonic-gate 		 * the lower half of cpu_intr_actv.
2160Sstevel@tonic-gate 		 */
2170Sstevel@tonic-gate 		uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1;
2180Sstevel@tonic-gate 
2190Sstevel@tonic-gate 		ASSERT(*refcntp > 0);
2200Sstevel@tonic-gate 
2210Sstevel@tonic-gate 		if (--(*refcntp) == 0)
2220Sstevel@tonic-gate 			cpu->cpu_intr_actv &= ~(1 << pil);
2230Sstevel@tonic-gate 	} else {
2240Sstevel@tonic-gate 		cpu->cpu_intr_actv &= ~(1 << pil);
2250Sstevel@tonic-gate 	}
2260Sstevel@tonic-gate 
2270Sstevel@tonic-gate 	ASSERT(mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] != 0);
2280Sstevel@tonic-gate 
229590Sesolom 	intrtime = tsc_read() - mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)];
230916Sschwartz 	mcpu->intrstat[pil][0] += intrtime;
231590Sesolom 	cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
2320Sstevel@tonic-gate 
2330Sstevel@tonic-gate 	/*
2340Sstevel@tonic-gate 	 * Check for lower-pil nested high-level interrupt beneath
2350Sstevel@tonic-gate 	 * current one.  If so, place a starting timestamp in its
2360Sstevel@tonic-gate 	 * pil_high_start entry.
2370Sstevel@tonic-gate 	 */
2380Sstevel@tonic-gate 	mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK;
2390Sstevel@tonic-gate 	if (mask != 0) {
2400Sstevel@tonic-gate 		int nestpil;
2410Sstevel@tonic-gate 
2420Sstevel@tonic-gate 		/*
2430Sstevel@tonic-gate 		 * find PIL of nested interrupt
2440Sstevel@tonic-gate 		 */
2450Sstevel@tonic-gate 		nestpil = bsrw_insn((uint16_t)mask);
2460Sstevel@tonic-gate 		ASSERT(nestpil < pil);
2470Sstevel@tonic-gate 		mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)] = tsc_read();
2480Sstevel@tonic-gate 		/*
2490Sstevel@tonic-gate 		 * (Another high-level interrupt is active below this one,
2500Sstevel@tonic-gate 		 * so there is no need to check for an interrupt
2510Sstevel@tonic-gate 		 * thread.  That will be done by the lowest priority
2520Sstevel@tonic-gate 		 * high-level interrupt active.)
2530Sstevel@tonic-gate 		 */
2540Sstevel@tonic-gate 	} else {
2550Sstevel@tonic-gate 		/*
2560Sstevel@tonic-gate 		 * Check to see if there is a low-level interrupt active.
2570Sstevel@tonic-gate 		 * If so, place a starting timestamp in the thread
2580Sstevel@tonic-gate 		 * structure.
2590Sstevel@tonic-gate 		 */
2600Sstevel@tonic-gate 		kthread_t *t = cpu->cpu_thread;
2610Sstevel@tonic-gate 
2620Sstevel@tonic-gate 		if (t->t_flag & T_INTR_THREAD)
2630Sstevel@tonic-gate 			t->t_intr_start = tsc_read();
2640Sstevel@tonic-gate 	}
2650Sstevel@tonic-gate 
2660Sstevel@tonic-gate 	mcpu->mcpu_pri = oldpil;
2670Sstevel@tonic-gate 	(void) (*setlvlx)(oldpil, vecnum);
2680Sstevel@tonic-gate 
2690Sstevel@tonic-gate 	return (cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK);
2700Sstevel@tonic-gate }
2710Sstevel@tonic-gate 
2720Sstevel@tonic-gate /*
2730Sstevel@tonic-gate  * Set up the cpu, thread and interrupt thread structures for
2740Sstevel@tonic-gate  * executing an interrupt thread.  The new stack pointer of the
2750Sstevel@tonic-gate  * interrupt thread (which *must* be switched to) is returned.
2760Sstevel@tonic-gate  */
2770Sstevel@tonic-gate caddr_t
2780Sstevel@tonic-gate intr_thread_prolog(struct cpu *cpu, caddr_t stackptr, uint_t pil)
2790Sstevel@tonic-gate {
2800Sstevel@tonic-gate 	struct machcpu *mcpu = &cpu->cpu_m;
2810Sstevel@tonic-gate 	kthread_t *t, *volatile it;
2820Sstevel@tonic-gate 
2830Sstevel@tonic-gate 	ASSERT(pil > 0);
2840Sstevel@tonic-gate 	ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
2850Sstevel@tonic-gate 	cpu->cpu_intr_actv |= (1 << pil);
2860Sstevel@tonic-gate 
2870Sstevel@tonic-gate 	/*
2880Sstevel@tonic-gate 	 * Get set to run an interrupt thread.
2890Sstevel@tonic-gate 	 * There should always be an interrupt thread, since we
2900Sstevel@tonic-gate 	 * allocate one for each level on each CPU.
2910Sstevel@tonic-gate 	 *
292989Sesolom 	 * t_intr_start could be zero due to cpu_intr_swtch_enter.
2930Sstevel@tonic-gate 	 */
2940Sstevel@tonic-gate 	t = cpu->cpu_thread;
295989Sesolom 	if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) {
296590Sesolom 		hrtime_t intrtime = tsc_read() - t->t_intr_start;
297916Sschwartz 		mcpu->intrstat[t->t_pil][0] += intrtime;
298590Sesolom 		cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
2990Sstevel@tonic-gate 		t->t_intr_start = 0;
3000Sstevel@tonic-gate 	}
3010Sstevel@tonic-gate 
3020Sstevel@tonic-gate 	ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr);
3030Sstevel@tonic-gate 
3040Sstevel@tonic-gate 	t->t_sp = (uintptr_t)stackptr;	/* mark stack in curthread for resume */
3050Sstevel@tonic-gate 
3060Sstevel@tonic-gate 	/*
3070Sstevel@tonic-gate 	 * unlink the interrupt thread off the cpu
308989Sesolom 	 *
309989Sesolom 	 * Note that the code in kcpc_overflow_intr -relies- on the
310989Sesolom 	 * ordering of events here - in particular that t->t_lwp of
311989Sesolom 	 * the interrupt thread is set to the pinned thread *before*
312989Sesolom 	 * curthread is changed.
3130Sstevel@tonic-gate 	 */
3140Sstevel@tonic-gate 	it = cpu->cpu_intr_thread;
3150Sstevel@tonic-gate 	cpu->cpu_intr_thread = it->t_link;
3160Sstevel@tonic-gate 	it->t_intr = t;
3170Sstevel@tonic-gate 	it->t_lwp = t->t_lwp;
3180Sstevel@tonic-gate 
3190Sstevel@tonic-gate 	/*
3200Sstevel@tonic-gate 	 * (threads on the interrupt thread free list could have state
3210Sstevel@tonic-gate 	 * preset to TS_ONPROC, but it helps in debugging if
3220Sstevel@tonic-gate 	 * they're TS_FREE.)
3230Sstevel@tonic-gate 	 */
3240Sstevel@tonic-gate 	it->t_state = TS_ONPROC;
3250Sstevel@tonic-gate 
3260Sstevel@tonic-gate 	cpu->cpu_thread = it;		/* new curthread on this cpu */
3270Sstevel@tonic-gate 	it->t_pil = (uchar_t)pil;
3280Sstevel@tonic-gate 	it->t_pri = intr_pri + (pri_t)pil;
3290Sstevel@tonic-gate 	it->t_intr_start = tsc_read();
3300Sstevel@tonic-gate 
3310Sstevel@tonic-gate 	return (it->t_stk);
3320Sstevel@tonic-gate }
3330Sstevel@tonic-gate 
3340Sstevel@tonic-gate 
3350Sstevel@tonic-gate #ifdef DEBUG
3360Sstevel@tonic-gate int intr_thread_cnt;
3370Sstevel@tonic-gate #endif
3380Sstevel@tonic-gate 
3390Sstevel@tonic-gate /*
3400Sstevel@tonic-gate  * Called with interrupts disabled
3410Sstevel@tonic-gate  */
3420Sstevel@tonic-gate void
3430Sstevel@tonic-gate intr_thread_epilog(struct cpu *cpu, uint_t vec, uint_t oldpil)
3440Sstevel@tonic-gate {
3450Sstevel@tonic-gate 	struct machcpu *mcpu = &cpu->cpu_m;
3460Sstevel@tonic-gate 	kthread_t *t;
3470Sstevel@tonic-gate 	kthread_t *it = cpu->cpu_thread;	/* curthread */
3480Sstevel@tonic-gate 	uint_t pil, basespl;
349590Sesolom 	hrtime_t intrtime;
3500Sstevel@tonic-gate 
3510Sstevel@tonic-gate 	pil = it->t_pil;
3520Sstevel@tonic-gate 	cpu->cpu_stats.sys.intr[pil - 1]++;
3530Sstevel@tonic-gate 
3540Sstevel@tonic-gate 	ASSERT(it->t_intr_start != 0);
355590Sesolom 	intrtime = tsc_read() - it->t_intr_start;
356916Sschwartz 	mcpu->intrstat[pil][0] += intrtime;
357590Sesolom 	cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
3580Sstevel@tonic-gate 
3590Sstevel@tonic-gate 	ASSERT(cpu->cpu_intr_actv & (1 << pil));
3600Sstevel@tonic-gate 	cpu->cpu_intr_actv &= ~(1 << pil);
3610Sstevel@tonic-gate 
3620Sstevel@tonic-gate 	/*
3630Sstevel@tonic-gate 	 * If there is still an interrupted thread underneath this one
3640Sstevel@tonic-gate 	 * then the interrupt was never blocked and the return is
3650Sstevel@tonic-gate 	 * fairly simple.  Otherwise it isn't.
3660Sstevel@tonic-gate 	 */
3670Sstevel@tonic-gate 	if ((t = it->t_intr) == NULL) {
3680Sstevel@tonic-gate 		/*
3690Sstevel@tonic-gate 		 * The interrupted thread is no longer pinned underneath
3700Sstevel@tonic-gate 		 * the interrupt thread.  This means the interrupt must
3710Sstevel@tonic-gate 		 * have blocked, and the interrupted thread has been
3720Sstevel@tonic-gate 		 * unpinned, and has probably been running around the
3730Sstevel@tonic-gate 		 * system for a while.
3740Sstevel@tonic-gate 		 *
3750Sstevel@tonic-gate 		 * Since there is no longer a thread under this one, put
3760Sstevel@tonic-gate 		 * this interrupt thread back on the CPU's free list and
3770Sstevel@tonic-gate 		 * resume the idle thread which will dispatch the next
3780Sstevel@tonic-gate 		 * thread to run.
3790Sstevel@tonic-gate 		 */
3800Sstevel@tonic-gate #ifdef DEBUG
3810Sstevel@tonic-gate 		intr_thread_cnt++;
3820Sstevel@tonic-gate #endif
3830Sstevel@tonic-gate 		cpu->cpu_stats.sys.intrblk++;
3840Sstevel@tonic-gate 		/*
3850Sstevel@tonic-gate 		 * Set CPU's base SPL based on active interrupts bitmask
3860Sstevel@tonic-gate 		 */
3870Sstevel@tonic-gate 		set_base_spl();
3880Sstevel@tonic-gate 		basespl = cpu->cpu_base_spl;
3890Sstevel@tonic-gate 		mcpu->mcpu_pri = basespl;
3900Sstevel@tonic-gate 		(*setlvlx)(basespl, vec);
3910Sstevel@tonic-gate 		(void) splhigh();
3920Sstevel@tonic-gate 		it->t_state = TS_FREE;
3930Sstevel@tonic-gate 		/*
3940Sstevel@tonic-gate 		 * Return interrupt thread to pool
3950Sstevel@tonic-gate 		 */
3960Sstevel@tonic-gate 		it->t_link = cpu->cpu_intr_thread;
3970Sstevel@tonic-gate 		cpu->cpu_intr_thread = it;
3980Sstevel@tonic-gate 		swtch();
3990Sstevel@tonic-gate 		/*NOTREACHED*/
4000Sstevel@tonic-gate 	}
4010Sstevel@tonic-gate 
4020Sstevel@tonic-gate 	/*
4030Sstevel@tonic-gate 	 * Return interrupt thread to the pool
4040Sstevel@tonic-gate 	 */
4050Sstevel@tonic-gate 	it->t_link = cpu->cpu_intr_thread;
4060Sstevel@tonic-gate 	cpu->cpu_intr_thread = it;
4070Sstevel@tonic-gate 	it->t_state = TS_FREE;
4080Sstevel@tonic-gate 
4090Sstevel@tonic-gate 	basespl = cpu->cpu_base_spl;
4100Sstevel@tonic-gate 	pil = MAX(oldpil, basespl);
4110Sstevel@tonic-gate 	mcpu->mcpu_pri = pil;
4120Sstevel@tonic-gate 	(*setlvlx)(pil, vec);
4130Sstevel@tonic-gate 	t->t_intr_start = tsc_read();
4140Sstevel@tonic-gate 	cpu->cpu_thread = t;
4150Sstevel@tonic-gate }
4160Sstevel@tonic-gate 
417916Sschwartz /*
418916Sschwartz  * Called with interrupts disabled by an interrupt thread to determine
419916Sschwartz  * how much time has elapsed. See interrupt.s:intr_get_time() for detailed
420916Sschwartz  * theory of operation.
421916Sschwartz  */
422916Sschwartz uint64_t
423916Sschwartz intr_thread_get_time(struct cpu *cpu)
424916Sschwartz {
425916Sschwartz 	struct machcpu *mcpu = &cpu->cpu_m;
426916Sschwartz 	kthread_t *t = cpu->cpu_thread;
427916Sschwartz 	uint64_t time, delta, ret;
428916Sschwartz 	uint_t pil = t->t_pil;
429916Sschwartz 
430916Sschwartz 	ASSERT((cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK) == 0);
431916Sschwartz 	ASSERT(t->t_flag & T_INTR_THREAD);
432916Sschwartz 	ASSERT(pil != 0);
433916Sschwartz 	ASSERT(t->t_intr_start != 0);
434916Sschwartz 
435916Sschwartz 	time = tsc_read();
436916Sschwartz 	delta = time - t->t_intr_start;
437916Sschwartz 	t->t_intr_start = time;
438916Sschwartz 
439916Sschwartz 	time = mcpu->intrstat[pil][0] + delta;
440916Sschwartz 	ret = time - mcpu->intrstat[pil][1];
441916Sschwartz 	mcpu->intrstat[pil][0] = time;
442916Sschwartz 	mcpu->intrstat[pil][1] = time;
443916Sschwartz 
444916Sschwartz 	return (ret);
445916Sschwartz }
446916Sschwartz 
4470Sstevel@tonic-gate caddr_t
4480Sstevel@tonic-gate dosoftint_prolog(
4490Sstevel@tonic-gate 	struct cpu *cpu,
4500Sstevel@tonic-gate 	caddr_t stackptr,
4510Sstevel@tonic-gate 	uint32_t st_pending,
4520Sstevel@tonic-gate 	uint_t oldpil)
4530Sstevel@tonic-gate {
4540Sstevel@tonic-gate 	kthread_t *t, *volatile it;
4550Sstevel@tonic-gate 	struct machcpu *mcpu = &cpu->cpu_m;
4560Sstevel@tonic-gate 	uint_t pil;
4570Sstevel@tonic-gate 
4580Sstevel@tonic-gate top:
4590Sstevel@tonic-gate 	ASSERT(st_pending == mcpu->mcpu_softinfo.st_pending);
4600Sstevel@tonic-gate 
4610Sstevel@tonic-gate 	pil = bsrw_insn((uint16_t)st_pending);
4620Sstevel@tonic-gate 	if (pil <= oldpil || pil <= cpu->cpu_base_spl)
4630Sstevel@tonic-gate 		return (0);
4640Sstevel@tonic-gate 
4650Sstevel@tonic-gate 	/*
4660Sstevel@tonic-gate 	 * XX64	Sigh.
4670Sstevel@tonic-gate 	 *
4680Sstevel@tonic-gate 	 * This is a transliteration of the i386 assembler code for
4690Sstevel@tonic-gate 	 * soft interrupts.  One question is "why does this need
4700Sstevel@tonic-gate 	 * to be atomic?"  One possible race is -other- processors
4710Sstevel@tonic-gate 	 * posting soft interrupts to us in set_pending() i.e. the
4720Sstevel@tonic-gate 	 * CPU might get preempted just after the address computation,
4730Sstevel@tonic-gate 	 * but just before the atomic transaction, so another CPU would
4740Sstevel@tonic-gate 	 * actually set the original CPU's st_pending bit.  However,
4750Sstevel@tonic-gate 	 * it looks like it would be simpler to disable preemption there.
4760Sstevel@tonic-gate 	 * Are there other races for which preemption control doesn't work?
4770Sstevel@tonic-gate 	 *
4780Sstevel@tonic-gate 	 * The i386 assembler version -also- checks to see if the bit
4790Sstevel@tonic-gate 	 * being cleared was actually set; if it wasn't, it rechecks
4800Sstevel@tonic-gate 	 * for more.  This seems a bit strange, as the only code that
4810Sstevel@tonic-gate 	 * ever clears the bit is -this- code running with interrupts
4820Sstevel@tonic-gate 	 * disabled on -this- CPU.  This code would probably be cheaper:
4830Sstevel@tonic-gate 	 *
4840Sstevel@tonic-gate 	 * atomic_and_32((uint32_t *)&mcpu->mcpu_softinfo.st_pending,
4850Sstevel@tonic-gate 	 *   ~(1 << pil));
4860Sstevel@tonic-gate 	 *
4870Sstevel@tonic-gate 	 * and t->t_preempt--/++ around set_pending() even cheaper,
4880Sstevel@tonic-gate 	 * but at this point, correctness is critical, so we slavishly
4890Sstevel@tonic-gate 	 * emulate the i386 port.
4900Sstevel@tonic-gate 	 */
4910Sstevel@tonic-gate 	if (atomic_btr32((uint32_t *)&mcpu->mcpu_softinfo.st_pending, pil)
4920Sstevel@tonic-gate 	    == 0) {
4930Sstevel@tonic-gate 		st_pending = mcpu->mcpu_softinfo.st_pending;
4940Sstevel@tonic-gate 		goto top;
4950Sstevel@tonic-gate 	}
4960Sstevel@tonic-gate 
4970Sstevel@tonic-gate 	mcpu->mcpu_pri = pil;
4980Sstevel@tonic-gate 	(*setspl)(pil);
4990Sstevel@tonic-gate 
5000Sstevel@tonic-gate 	/*
5010Sstevel@tonic-gate 	 * Get set to run interrupt thread.
5020Sstevel@tonic-gate 	 * There should always be an interrupt thread since we
5030Sstevel@tonic-gate 	 * allocate one for each level on the CPU.
5040Sstevel@tonic-gate 	 */
5050Sstevel@tonic-gate 	it = cpu->cpu_intr_thread;
5060Sstevel@tonic-gate 	cpu->cpu_intr_thread = it->t_link;
5070Sstevel@tonic-gate 
508989Sesolom 	/* t_intr_start could be zero due to cpu_intr_swtch_enter. */
509989Sesolom 	t = cpu->cpu_thread;
510989Sesolom 	if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) {
511989Sesolom 		hrtime_t intrtime = tsc_read() - t->t_intr_start;
512989Sesolom 		mcpu->intrstat[pil][0] += intrtime;
513989Sesolom 		cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
514989Sesolom 		t->t_intr_start = 0;
515989Sesolom 	}
516989Sesolom 
5170Sstevel@tonic-gate 	/*
5180Sstevel@tonic-gate 	 * Note that the code in kcpc_overflow_intr -relies- on the
5190Sstevel@tonic-gate 	 * ordering of events here - in particular that t->t_lwp of
5200Sstevel@tonic-gate 	 * the interrupt thread is set to the pinned thread *before*
521989Sesolom 	 * curthread is changed.
5220Sstevel@tonic-gate 	 */
5230Sstevel@tonic-gate 	it->t_lwp = t->t_lwp;
5240Sstevel@tonic-gate 	it->t_state = TS_ONPROC;
5250Sstevel@tonic-gate 
5260Sstevel@tonic-gate 	/*
5270Sstevel@tonic-gate 	 * Push interrupted thread onto list from new thread.
5280Sstevel@tonic-gate 	 * Set the new thread as the current one.
5290Sstevel@tonic-gate 	 * Set interrupted thread's T_SP because if it is the idle thread,
5300Sstevel@tonic-gate 	 * resume() may use that stack between threads.
5310Sstevel@tonic-gate 	 */
5320Sstevel@tonic-gate 
5330Sstevel@tonic-gate 	ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr);
5340Sstevel@tonic-gate 	t->t_sp = (uintptr_t)stackptr;
5350Sstevel@tonic-gate 
5360Sstevel@tonic-gate 	it->t_intr = t;
5370Sstevel@tonic-gate 	cpu->cpu_thread = it;
5380Sstevel@tonic-gate 
5390Sstevel@tonic-gate 	/*
5400Sstevel@tonic-gate 	 * Set bit for this pil in CPU's interrupt active bitmask.
5410Sstevel@tonic-gate 	 */
5420Sstevel@tonic-gate 	ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
5430Sstevel@tonic-gate 	cpu->cpu_intr_actv |= (1 << pil);
5440Sstevel@tonic-gate 
5450Sstevel@tonic-gate 	/*
5460Sstevel@tonic-gate 	 * Initialize thread priority level from intr_pri
5470Sstevel@tonic-gate 	 */
5480Sstevel@tonic-gate 	it->t_pil = (uchar_t)pil;
5490Sstevel@tonic-gate 	it->t_pri = (pri_t)pil + intr_pri;
5500Sstevel@tonic-gate 	it->t_intr_start = tsc_read();
5510Sstevel@tonic-gate 
5520Sstevel@tonic-gate 	return (it->t_stk);
5530Sstevel@tonic-gate }
5540Sstevel@tonic-gate 
5550Sstevel@tonic-gate void
5560Sstevel@tonic-gate dosoftint_epilog(struct cpu *cpu, uint_t oldpil)
5570Sstevel@tonic-gate {
5580Sstevel@tonic-gate 	struct machcpu *mcpu = &cpu->cpu_m;
5590Sstevel@tonic-gate 	kthread_t *t, *it;
5600Sstevel@tonic-gate 	uint_t pil, basespl;
561590Sesolom 	hrtime_t intrtime;
5620Sstevel@tonic-gate 
5630Sstevel@tonic-gate 	it = cpu->cpu_thread;
5640Sstevel@tonic-gate 	pil = it->t_pil;
5650Sstevel@tonic-gate 
5660Sstevel@tonic-gate 	cpu->cpu_stats.sys.intr[pil - 1]++;
5670Sstevel@tonic-gate 
5680Sstevel@tonic-gate 	ASSERT(cpu->cpu_intr_actv & (1 << pil));
5690Sstevel@tonic-gate 	cpu->cpu_intr_actv &= ~(1 << pil);
570590Sesolom 	intrtime = tsc_read() - it->t_intr_start;
571916Sschwartz 	mcpu->intrstat[pil][0] += intrtime;
572590Sesolom 	cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
5730Sstevel@tonic-gate 
5740Sstevel@tonic-gate 	/*
5750Sstevel@tonic-gate 	 * If there is still an interrupted thread underneath this one
5760Sstevel@tonic-gate 	 * then the interrupt was never blocked and the return is
5770Sstevel@tonic-gate 	 * fairly simple.  Otherwise it isn't.
5780Sstevel@tonic-gate 	 */
5790Sstevel@tonic-gate 	if ((t = it->t_intr) == NULL) {
5800Sstevel@tonic-gate 		/*
5810Sstevel@tonic-gate 		 * Put thread back on the interrupt thread list.
5820Sstevel@tonic-gate 		 * This was an interrupt thread, so set CPU's base SPL.
5830Sstevel@tonic-gate 		 */
5840Sstevel@tonic-gate 		set_base_spl();
5850Sstevel@tonic-gate 		it->t_state = TS_FREE;
5860Sstevel@tonic-gate 		it->t_link = cpu->cpu_intr_thread;
5870Sstevel@tonic-gate 		cpu->cpu_intr_thread = it;
5880Sstevel@tonic-gate 		(void) splhigh();
5890Sstevel@tonic-gate 		swtch();
5900Sstevel@tonic-gate 		/*NOTREACHED*/
5910Sstevel@tonic-gate 	}
5920Sstevel@tonic-gate 	it->t_link = cpu->cpu_intr_thread;
5930Sstevel@tonic-gate 	cpu->cpu_intr_thread = it;
5940Sstevel@tonic-gate 	it->t_state = TS_FREE;
5950Sstevel@tonic-gate 	cpu->cpu_thread = t;
5960Sstevel@tonic-gate 	if (t->t_flag & T_INTR_THREAD)
5970Sstevel@tonic-gate 		t->t_intr_start = tsc_read();
5980Sstevel@tonic-gate 	basespl = cpu->cpu_base_spl;
5990Sstevel@tonic-gate 	pil = MAX(oldpil, basespl);
6000Sstevel@tonic-gate 	mcpu->mcpu_pri = pil;
6010Sstevel@tonic-gate 	(*setspl)(pil);
6020Sstevel@tonic-gate }
6030Sstevel@tonic-gate 
6040Sstevel@tonic-gate /*
6050Sstevel@tonic-gate  * Make the interrupted thread 'to' be runnable.
6060Sstevel@tonic-gate  *
6070Sstevel@tonic-gate  * Since t->t_sp has already been saved, t->t_pc is all
6080Sstevel@tonic-gate  * that needs to be set in this function.
6090Sstevel@tonic-gate  *
6100Sstevel@tonic-gate  * Returns the interrupt level of the interrupt thread.
6110Sstevel@tonic-gate  */
6120Sstevel@tonic-gate int
6130Sstevel@tonic-gate intr_passivate(
6140Sstevel@tonic-gate 	kthread_t *it,		/* interrupt thread */
6150Sstevel@tonic-gate 	kthread_t *t)		/* interrupted thread */
6160Sstevel@tonic-gate {
6170Sstevel@tonic-gate 	extern void _sys_rtt();
6180Sstevel@tonic-gate 
6190Sstevel@tonic-gate 	ASSERT(it->t_flag & T_INTR_THREAD);
6200Sstevel@tonic-gate 	ASSERT(SA(t->t_sp) == t->t_sp);
6210Sstevel@tonic-gate 
6220Sstevel@tonic-gate 	t->t_pc = (uintptr_t)_sys_rtt;
6230Sstevel@tonic-gate 	return (it->t_pil);
6240Sstevel@tonic-gate }
6250Sstevel@tonic-gate 
6260Sstevel@tonic-gate #endif	/* __amd64 */
6270Sstevel@tonic-gate 
6280Sstevel@tonic-gate /*
6290Sstevel@tonic-gate  * Create interrupt kstats for this CPU.
6300Sstevel@tonic-gate  */
6310Sstevel@tonic-gate void
6320Sstevel@tonic-gate cpu_create_intrstat(cpu_t *cp)
6330Sstevel@tonic-gate {
6340Sstevel@tonic-gate 	int		i;
6350Sstevel@tonic-gate 	kstat_t		*intr_ksp;
6360Sstevel@tonic-gate 	kstat_named_t	*knp;
6370Sstevel@tonic-gate 	char		name[KSTAT_STRLEN];
6380Sstevel@tonic-gate 	zoneid_t	zoneid;
6390Sstevel@tonic-gate 
6400Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
6410Sstevel@tonic-gate 
6420Sstevel@tonic-gate 	if (pool_pset_enabled())
6430Sstevel@tonic-gate 		zoneid = GLOBAL_ZONEID;
6440Sstevel@tonic-gate 	else
6450Sstevel@tonic-gate 		zoneid = ALL_ZONES;
6460Sstevel@tonic-gate 
6470Sstevel@tonic-gate 	intr_ksp = kstat_create_zone("cpu", cp->cpu_id, "intrstat", "misc",
6480Sstevel@tonic-gate 	    KSTAT_TYPE_NAMED, PIL_MAX * 2, NULL, zoneid);
6490Sstevel@tonic-gate 
6500Sstevel@tonic-gate 	/*
6510Sstevel@tonic-gate 	 * Initialize each PIL's named kstat
6520Sstevel@tonic-gate 	 */
6530Sstevel@tonic-gate 	if (intr_ksp != NULL) {
6540Sstevel@tonic-gate 		intr_ksp->ks_update = cpu_kstat_intrstat_update;
6550Sstevel@tonic-gate 		knp = (kstat_named_t *)intr_ksp->ks_data;
6560Sstevel@tonic-gate 		intr_ksp->ks_private = cp;
6570Sstevel@tonic-gate 		for (i = 0; i < PIL_MAX; i++) {
6580Sstevel@tonic-gate 			(void) snprintf(name, KSTAT_STRLEN, "level-%d-time",
6590Sstevel@tonic-gate 			    i + 1);
6600Sstevel@tonic-gate 			kstat_named_init(&knp[i * 2], name, KSTAT_DATA_UINT64);
6610Sstevel@tonic-gate 			(void) snprintf(name, KSTAT_STRLEN, "level-%d-count",
6620Sstevel@tonic-gate 			    i + 1);
6630Sstevel@tonic-gate 			kstat_named_init(&knp[(i * 2) + 1], name,
6640Sstevel@tonic-gate 			    KSTAT_DATA_UINT64);
6650Sstevel@tonic-gate 		}
6660Sstevel@tonic-gate 		kstat_install(intr_ksp);
6670Sstevel@tonic-gate 	}
6680Sstevel@tonic-gate }
6690Sstevel@tonic-gate 
6700Sstevel@tonic-gate /*
6710Sstevel@tonic-gate  * Delete interrupt kstats for this CPU.
6720Sstevel@tonic-gate  */
6730Sstevel@tonic-gate void
6740Sstevel@tonic-gate cpu_delete_intrstat(cpu_t *cp)
6750Sstevel@tonic-gate {
6760Sstevel@tonic-gate 	kstat_delete_byname_zone("cpu", cp->cpu_id, "intrstat", ALL_ZONES);
6770Sstevel@tonic-gate }
6780Sstevel@tonic-gate 
6790Sstevel@tonic-gate /*
6800Sstevel@tonic-gate  * Convert interrupt statistics from CPU ticks to nanoseconds and
6810Sstevel@tonic-gate  * update kstat.
6820Sstevel@tonic-gate  */
6830Sstevel@tonic-gate int
6840Sstevel@tonic-gate cpu_kstat_intrstat_update(kstat_t *ksp, int rw)
6850Sstevel@tonic-gate {
6860Sstevel@tonic-gate 	kstat_named_t	*knp = ksp->ks_data;
6870Sstevel@tonic-gate 	cpu_t		*cpup = (cpu_t *)ksp->ks_private;
6880Sstevel@tonic-gate 	int		i;
6890Sstevel@tonic-gate 	hrtime_t	hrt;
6900Sstevel@tonic-gate 
6910Sstevel@tonic-gate 	if (rw == KSTAT_WRITE)
6920Sstevel@tonic-gate 		return (EACCES);
6930Sstevel@tonic-gate 
6940Sstevel@tonic-gate 	for (i = 0; i < PIL_MAX; i++) {
695916Sschwartz 		hrt = (hrtime_t)cpup->cpu_m.intrstat[i + 1][0];
6960Sstevel@tonic-gate 		tsc_scalehrtime(&hrt);
6970Sstevel@tonic-gate 		knp[i * 2].value.ui64 = (uint64_t)hrt;
6980Sstevel@tonic-gate 		knp[(i * 2) + 1].value.ui64 = cpup->cpu_stats.sys.intr[i];
6990Sstevel@tonic-gate 	}
7000Sstevel@tonic-gate 
7010Sstevel@tonic-gate 	return (0);
7020Sstevel@tonic-gate }
7030Sstevel@tonic-gate 
7040Sstevel@tonic-gate /*
7050Sstevel@tonic-gate  * An interrupt thread is ending a time slice, so compute the interval it
7060Sstevel@tonic-gate  * ran for and update the statistic for its PIL.
7070Sstevel@tonic-gate  */
7080Sstevel@tonic-gate void
7090Sstevel@tonic-gate cpu_intr_swtch_enter(kthread_id_t t)
7100Sstevel@tonic-gate {
7110Sstevel@tonic-gate 	uint64_t	interval;
7120Sstevel@tonic-gate 	uint64_t	start;
713590Sesolom 	cpu_t		*cpu;
7140Sstevel@tonic-gate 
7150Sstevel@tonic-gate 	ASSERT((t->t_flag & T_INTR_THREAD) != 0);
7160Sstevel@tonic-gate 	ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL);
7170Sstevel@tonic-gate 
7180Sstevel@tonic-gate 	/*
7190Sstevel@tonic-gate 	 * We could be here with a zero timestamp. This could happen if:
7200Sstevel@tonic-gate 	 * an interrupt thread which no longer has a pinned thread underneath
7210Sstevel@tonic-gate 	 * it (i.e. it blocked at some point in its past) has finished running
7220Sstevel@tonic-gate 	 * its handler. intr_thread() updated the interrupt statistic for its
7230Sstevel@tonic-gate 	 * PIL and zeroed its timestamp. Since there was no pinned thread to
7240Sstevel@tonic-gate 	 * return to, swtch() gets called and we end up here.
725590Sesolom 	 *
726590Sesolom 	 * Note that we use atomic ops below (cas64 and atomic_add_64), which
727590Sesolom 	 * we don't use in the functions above, because we're not called
728590Sesolom 	 * with interrupts blocked, but the epilog/prolog functions are.
7290Sstevel@tonic-gate 	 */
7300Sstevel@tonic-gate 	if (t->t_intr_start) {
7310Sstevel@tonic-gate 		do {
7320Sstevel@tonic-gate 			start = t->t_intr_start;
7330Sstevel@tonic-gate 			interval = tsc_read() - start;
7340Sstevel@tonic-gate 		} while (cas64(&t->t_intr_start, start, 0) != start);
735590Sesolom 		cpu = CPU;
736916Sschwartz 		cpu->cpu_m.intrstat[t->t_pil][0] += interval;
737590Sesolom 
738590Sesolom 		atomic_add_64((uint64_t *)&cpu->cpu_intracct[cpu->cpu_mstate],
739590Sesolom 		    interval);
7400Sstevel@tonic-gate 	} else
7410Sstevel@tonic-gate 		ASSERT(t->t_intr == NULL);
7420Sstevel@tonic-gate }
7430Sstevel@tonic-gate 
7440Sstevel@tonic-gate /*
7450Sstevel@tonic-gate  * An interrupt thread is returning from swtch(). Place a starting timestamp
7460Sstevel@tonic-gate  * in its thread structure.
7470Sstevel@tonic-gate  */
7480Sstevel@tonic-gate void
7490Sstevel@tonic-gate cpu_intr_swtch_exit(kthread_id_t t)
7500Sstevel@tonic-gate {
7510Sstevel@tonic-gate 	uint64_t ts;
7520Sstevel@tonic-gate 
7530Sstevel@tonic-gate 	ASSERT((t->t_flag & T_INTR_THREAD) != 0);
7540Sstevel@tonic-gate 	ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL);
7550Sstevel@tonic-gate 
7560Sstevel@tonic-gate 	do {
7570Sstevel@tonic-gate 		ts = t->t_intr_start;
7580Sstevel@tonic-gate 	} while (cas64(&t->t_intr_start, ts, tsc_read()) != ts);
7590Sstevel@tonic-gate }
760