10Sstevel@tonic-gate /* 20Sstevel@tonic-gate * CDDL HEADER START 30Sstevel@tonic-gate * 40Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*1455Sandrei * Common Development and Distribution License (the "License"). 6*1455Sandrei * You may not use this file except in compliance with the License. 70Sstevel@tonic-gate * 80Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 90Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 100Sstevel@tonic-gate * See the License for the specific language governing permissions 110Sstevel@tonic-gate * and limitations under the License. 120Sstevel@tonic-gate * 130Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 140Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 150Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 160Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 170Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 180Sstevel@tonic-gate * 190Sstevel@tonic-gate * CDDL HEADER END 200Sstevel@tonic-gate */ 210Sstevel@tonic-gate /* 22*1455Sandrei * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 230Sstevel@tonic-gate * Use is subject to license terms. 240Sstevel@tonic-gate */ 250Sstevel@tonic-gate 260Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 270Sstevel@tonic-gate 280Sstevel@tonic-gate #include <sys/cpuvar.h> 290Sstevel@tonic-gate #include <sys/regset.h> 300Sstevel@tonic-gate #include <sys/psw.h> 310Sstevel@tonic-gate #include <sys/types.h> 320Sstevel@tonic-gate #include <sys/thread.h> 330Sstevel@tonic-gate #include <sys/systm.h> 340Sstevel@tonic-gate #include <sys/segments.h> 350Sstevel@tonic-gate #include <sys/pcb.h> 360Sstevel@tonic-gate #include <sys/trap.h> 370Sstevel@tonic-gate #include <sys/ftrace.h> 380Sstevel@tonic-gate #include <sys/traptrace.h> 390Sstevel@tonic-gate #include <sys/clock.h> 400Sstevel@tonic-gate #include <sys/panic.h> 410Sstevel@tonic-gate #include <sys/disp.h> 420Sstevel@tonic-gate #include <vm/seg_kp.h> 430Sstevel@tonic-gate #include <sys/stack.h> 440Sstevel@tonic-gate #include <sys/sysmacros.h> 450Sstevel@tonic-gate #include <sys/cmn_err.h> 460Sstevel@tonic-gate #include <sys/kstat.h> 470Sstevel@tonic-gate #include <sys/smp_impldefs.h> 480Sstevel@tonic-gate #include <sys/pool_pset.h> 490Sstevel@tonic-gate #include <sys/zone.h> 500Sstevel@tonic-gate #include <sys/bitmap.h> 510Sstevel@tonic-gate 520Sstevel@tonic-gate #if defined(__amd64) 530Sstevel@tonic-gate 540Sstevel@tonic-gate #if defined(__lint) 550Sstevel@tonic-gate /* 560Sstevel@tonic-gate * atomic_btr32() is a gcc __inline__ function, defined in <asm/bitmap.h> 570Sstevel@tonic-gate * For lint purposes, define it here. 580Sstevel@tonic-gate */ 590Sstevel@tonic-gate uint_t 600Sstevel@tonic-gate atomic_btr32(uint32_t *pending, uint_t pil) 610Sstevel@tonic-gate { 620Sstevel@tonic-gate return (*pending &= ~(1 << pil)); 630Sstevel@tonic-gate } 640Sstevel@tonic-gate #else 650Sstevel@tonic-gate 660Sstevel@tonic-gate extern uint_t atomic_btr32(uint32_t *pending, uint_t pil); 670Sstevel@tonic-gate 680Sstevel@tonic-gate #endif 690Sstevel@tonic-gate 700Sstevel@tonic-gate /* 710Sstevel@tonic-gate * This code is amd64-only for now, but as time permits, we should 720Sstevel@tonic-gate * use this on i386 too. 730Sstevel@tonic-gate */ 740Sstevel@tonic-gate 750Sstevel@tonic-gate /* 760Sstevel@tonic-gate * Some questions to ponder: 770Sstevel@tonic-gate * - in several of these routines, we make multiple calls to tsc_read() 780Sstevel@tonic-gate * without invoking functions .. couldn't we just reuse the same 790Sstevel@tonic-gate * timestamp sometimes? 800Sstevel@tonic-gate * - if we have the inline, we can probably make set_base_spl be a 810Sstevel@tonic-gate * C routine too. 820Sstevel@tonic-gate */ 830Sstevel@tonic-gate 840Sstevel@tonic-gate static uint_t 850Sstevel@tonic-gate bsrw_insn(uint16_t mask) 860Sstevel@tonic-gate { 870Sstevel@tonic-gate uint_t index = sizeof (mask) * NBBY - 1; 880Sstevel@tonic-gate 890Sstevel@tonic-gate ASSERT(mask != 0); 900Sstevel@tonic-gate 910Sstevel@tonic-gate while ((mask & (1 << index)) == 0) 920Sstevel@tonic-gate index--; 930Sstevel@tonic-gate return (index); 940Sstevel@tonic-gate } 950Sstevel@tonic-gate 960Sstevel@tonic-gate /* 970Sstevel@tonic-gate * Do all the work necessary to set up the cpu and thread structures 980Sstevel@tonic-gate * to dispatch a high-level interrupt. 990Sstevel@tonic-gate * 1000Sstevel@tonic-gate * Returns 0 if we're -not- already on the high-level interrupt stack, 1010Sstevel@tonic-gate * (and *must* switch to it), non-zero if we are already on that stack. 1020Sstevel@tonic-gate * 1030Sstevel@tonic-gate * Called with interrupts masked. 1040Sstevel@tonic-gate * The 'pil' is already set to the appropriate level for rp->r_trapno. 1050Sstevel@tonic-gate */ 1060Sstevel@tonic-gate int 1070Sstevel@tonic-gate hilevel_intr_prolog(struct cpu *cpu, uint_t pil, uint_t oldpil, struct regs *rp) 1080Sstevel@tonic-gate { 1090Sstevel@tonic-gate struct machcpu *mcpu = &cpu->cpu_m; 1100Sstevel@tonic-gate uint_t mask; 111590Sesolom hrtime_t intrtime; 1120Sstevel@tonic-gate 1130Sstevel@tonic-gate ASSERT(pil > LOCK_LEVEL); 1140Sstevel@tonic-gate 1150Sstevel@tonic-gate if (pil == CBE_HIGH_PIL) { 1160Sstevel@tonic-gate cpu->cpu_profile_pil = oldpil; 1170Sstevel@tonic-gate if (USERMODE(rp->r_cs)) { 1180Sstevel@tonic-gate cpu->cpu_profile_pc = 0; 1190Sstevel@tonic-gate cpu->cpu_profile_upc = rp->r_pc; 1200Sstevel@tonic-gate } else { 1210Sstevel@tonic-gate cpu->cpu_profile_pc = rp->r_pc; 1220Sstevel@tonic-gate cpu->cpu_profile_upc = 0; 1230Sstevel@tonic-gate } 1240Sstevel@tonic-gate } 1250Sstevel@tonic-gate 1260Sstevel@tonic-gate mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK; 1270Sstevel@tonic-gate if (mask != 0) { 1280Sstevel@tonic-gate int nestpil; 1290Sstevel@tonic-gate 1300Sstevel@tonic-gate /* 1310Sstevel@tonic-gate * We have interrupted another high-level interrupt. 1320Sstevel@tonic-gate * Load starting timestamp, compute interval, update 1330Sstevel@tonic-gate * cumulative counter. 1340Sstevel@tonic-gate */ 1350Sstevel@tonic-gate nestpil = bsrw_insn((uint16_t)mask); 1360Sstevel@tonic-gate ASSERT(nestpil < pil); 137590Sesolom intrtime = tsc_read() - 1380Sstevel@tonic-gate mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)]; 139916Sschwartz mcpu->intrstat[nestpil][0] += intrtime; 140590Sesolom cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 1410Sstevel@tonic-gate /* 1420Sstevel@tonic-gate * Another high-level interrupt is active below this one, so 1430Sstevel@tonic-gate * there is no need to check for an interrupt thread. That 1440Sstevel@tonic-gate * will be done by the lowest priority high-level interrupt 1450Sstevel@tonic-gate * active. 1460Sstevel@tonic-gate */ 1470Sstevel@tonic-gate } else { 1480Sstevel@tonic-gate kthread_t *t = cpu->cpu_thread; 1490Sstevel@tonic-gate 1500Sstevel@tonic-gate /* 1510Sstevel@tonic-gate * See if we are interrupting a low-level interrupt thread. 1520Sstevel@tonic-gate * If so, account for its time slice only if its time stamp 1530Sstevel@tonic-gate * is non-zero. 1540Sstevel@tonic-gate */ 1550Sstevel@tonic-gate if ((t->t_flag & T_INTR_THREAD) != 0 && t->t_intr_start != 0) { 156590Sesolom intrtime = tsc_read() - t->t_intr_start; 157916Sschwartz mcpu->intrstat[t->t_pil][0] += intrtime; 158590Sesolom cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 1590Sstevel@tonic-gate t->t_intr_start = 0; 1600Sstevel@tonic-gate } 1610Sstevel@tonic-gate } 1620Sstevel@tonic-gate 1630Sstevel@tonic-gate /* 1640Sstevel@tonic-gate * Store starting timestamp in CPU structure for this PIL. 1650Sstevel@tonic-gate */ 1660Sstevel@tonic-gate mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] = tsc_read(); 1670Sstevel@tonic-gate 1680Sstevel@tonic-gate ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0); 1690Sstevel@tonic-gate 1700Sstevel@tonic-gate if (pil == 15) { 1710Sstevel@tonic-gate /* 1720Sstevel@tonic-gate * To support reentrant level 15 interrupts, we maintain a 1730Sstevel@tonic-gate * recursion count in the top half of cpu_intr_actv. Only 1740Sstevel@tonic-gate * when this count hits zero do we clear the PIL 15 bit from 1750Sstevel@tonic-gate * the lower half of cpu_intr_actv. 1760Sstevel@tonic-gate */ 1770Sstevel@tonic-gate uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1; 1780Sstevel@tonic-gate (*refcntp)++; 1790Sstevel@tonic-gate } 1800Sstevel@tonic-gate 1810Sstevel@tonic-gate mask = cpu->cpu_intr_actv; 1820Sstevel@tonic-gate 1830Sstevel@tonic-gate cpu->cpu_intr_actv |= (1 << pil); 1840Sstevel@tonic-gate 1850Sstevel@tonic-gate return (mask & CPU_INTR_ACTV_HIGH_LEVEL_MASK); 1860Sstevel@tonic-gate } 1870Sstevel@tonic-gate 1880Sstevel@tonic-gate /* 1890Sstevel@tonic-gate * Does most of the work of returning from a high level interrupt. 1900Sstevel@tonic-gate * 1910Sstevel@tonic-gate * Returns 0 if there are no more high level interrupts (in which 1920Sstevel@tonic-gate * case we must switch back to the interrupted thread stack) or 1930Sstevel@tonic-gate * non-zero if there are more (in which case we should stay on it). 1940Sstevel@tonic-gate * 1950Sstevel@tonic-gate * Called with interrupts masked 1960Sstevel@tonic-gate */ 1970Sstevel@tonic-gate int 1980Sstevel@tonic-gate hilevel_intr_epilog(struct cpu *cpu, uint_t pil, uint_t oldpil, uint_t vecnum) 1990Sstevel@tonic-gate { 2000Sstevel@tonic-gate struct machcpu *mcpu = &cpu->cpu_m; 2010Sstevel@tonic-gate uint_t mask; 202590Sesolom hrtime_t intrtime; 2030Sstevel@tonic-gate 2040Sstevel@tonic-gate ASSERT(mcpu->mcpu_pri == pil); 2050Sstevel@tonic-gate 2060Sstevel@tonic-gate cpu->cpu_stats.sys.intr[pil - 1]++; 2070Sstevel@tonic-gate 2080Sstevel@tonic-gate ASSERT(cpu->cpu_intr_actv & (1 << pil)); 2090Sstevel@tonic-gate 2100Sstevel@tonic-gate if (pil == 15) { 2110Sstevel@tonic-gate /* 2120Sstevel@tonic-gate * To support reentrant level 15 interrupts, we maintain a 2130Sstevel@tonic-gate * recursion count in the top half of cpu_intr_actv. Only 2140Sstevel@tonic-gate * when this count hits zero do we clear the PIL 15 bit from 2150Sstevel@tonic-gate * the lower half of cpu_intr_actv. 2160Sstevel@tonic-gate */ 2170Sstevel@tonic-gate uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1; 2180Sstevel@tonic-gate 2190Sstevel@tonic-gate ASSERT(*refcntp > 0); 2200Sstevel@tonic-gate 2210Sstevel@tonic-gate if (--(*refcntp) == 0) 2220Sstevel@tonic-gate cpu->cpu_intr_actv &= ~(1 << pil); 2230Sstevel@tonic-gate } else { 2240Sstevel@tonic-gate cpu->cpu_intr_actv &= ~(1 << pil); 2250Sstevel@tonic-gate } 2260Sstevel@tonic-gate 2270Sstevel@tonic-gate ASSERT(mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] != 0); 2280Sstevel@tonic-gate 229590Sesolom intrtime = tsc_read() - mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)]; 230916Sschwartz mcpu->intrstat[pil][0] += intrtime; 231590Sesolom cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 2320Sstevel@tonic-gate 2330Sstevel@tonic-gate /* 2340Sstevel@tonic-gate * Check for lower-pil nested high-level interrupt beneath 2350Sstevel@tonic-gate * current one. If so, place a starting timestamp in its 2360Sstevel@tonic-gate * pil_high_start entry. 2370Sstevel@tonic-gate */ 2380Sstevel@tonic-gate mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK; 2390Sstevel@tonic-gate if (mask != 0) { 2400Sstevel@tonic-gate int nestpil; 2410Sstevel@tonic-gate 2420Sstevel@tonic-gate /* 2430Sstevel@tonic-gate * find PIL of nested interrupt 2440Sstevel@tonic-gate */ 2450Sstevel@tonic-gate nestpil = bsrw_insn((uint16_t)mask); 2460Sstevel@tonic-gate ASSERT(nestpil < pil); 2470Sstevel@tonic-gate mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)] = tsc_read(); 2480Sstevel@tonic-gate /* 2490Sstevel@tonic-gate * (Another high-level interrupt is active below this one, 2500Sstevel@tonic-gate * so there is no need to check for an interrupt 2510Sstevel@tonic-gate * thread. That will be done by the lowest priority 2520Sstevel@tonic-gate * high-level interrupt active.) 2530Sstevel@tonic-gate */ 2540Sstevel@tonic-gate } else { 2550Sstevel@tonic-gate /* 2560Sstevel@tonic-gate * Check to see if there is a low-level interrupt active. 2570Sstevel@tonic-gate * If so, place a starting timestamp in the thread 2580Sstevel@tonic-gate * structure. 2590Sstevel@tonic-gate */ 2600Sstevel@tonic-gate kthread_t *t = cpu->cpu_thread; 2610Sstevel@tonic-gate 2620Sstevel@tonic-gate if (t->t_flag & T_INTR_THREAD) 2630Sstevel@tonic-gate t->t_intr_start = tsc_read(); 2640Sstevel@tonic-gate } 2650Sstevel@tonic-gate 2660Sstevel@tonic-gate mcpu->mcpu_pri = oldpil; 2670Sstevel@tonic-gate (void) (*setlvlx)(oldpil, vecnum); 2680Sstevel@tonic-gate 2690Sstevel@tonic-gate return (cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK); 2700Sstevel@tonic-gate } 2710Sstevel@tonic-gate 2720Sstevel@tonic-gate /* 2730Sstevel@tonic-gate * Set up the cpu, thread and interrupt thread structures for 2740Sstevel@tonic-gate * executing an interrupt thread. The new stack pointer of the 2750Sstevel@tonic-gate * interrupt thread (which *must* be switched to) is returned. 2760Sstevel@tonic-gate */ 2770Sstevel@tonic-gate caddr_t 2780Sstevel@tonic-gate intr_thread_prolog(struct cpu *cpu, caddr_t stackptr, uint_t pil) 2790Sstevel@tonic-gate { 2800Sstevel@tonic-gate struct machcpu *mcpu = &cpu->cpu_m; 2810Sstevel@tonic-gate kthread_t *t, *volatile it; 2820Sstevel@tonic-gate 2830Sstevel@tonic-gate ASSERT(pil > 0); 2840Sstevel@tonic-gate ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0); 2850Sstevel@tonic-gate cpu->cpu_intr_actv |= (1 << pil); 2860Sstevel@tonic-gate 2870Sstevel@tonic-gate /* 2880Sstevel@tonic-gate * Get set to run an interrupt thread. 2890Sstevel@tonic-gate * There should always be an interrupt thread, since we 2900Sstevel@tonic-gate * allocate one for each level on each CPU. 2910Sstevel@tonic-gate * 292989Sesolom * t_intr_start could be zero due to cpu_intr_swtch_enter. 2930Sstevel@tonic-gate */ 2940Sstevel@tonic-gate t = cpu->cpu_thread; 295989Sesolom if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) { 296590Sesolom hrtime_t intrtime = tsc_read() - t->t_intr_start; 297916Sschwartz mcpu->intrstat[t->t_pil][0] += intrtime; 298590Sesolom cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 2990Sstevel@tonic-gate t->t_intr_start = 0; 3000Sstevel@tonic-gate } 3010Sstevel@tonic-gate 3020Sstevel@tonic-gate ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr); 3030Sstevel@tonic-gate 3040Sstevel@tonic-gate t->t_sp = (uintptr_t)stackptr; /* mark stack in curthread for resume */ 3050Sstevel@tonic-gate 3060Sstevel@tonic-gate /* 3070Sstevel@tonic-gate * unlink the interrupt thread off the cpu 308989Sesolom * 309989Sesolom * Note that the code in kcpc_overflow_intr -relies- on the 310989Sesolom * ordering of events here - in particular that t->t_lwp of 311989Sesolom * the interrupt thread is set to the pinned thread *before* 312989Sesolom * curthread is changed. 3130Sstevel@tonic-gate */ 3140Sstevel@tonic-gate it = cpu->cpu_intr_thread; 3150Sstevel@tonic-gate cpu->cpu_intr_thread = it->t_link; 3160Sstevel@tonic-gate it->t_intr = t; 3170Sstevel@tonic-gate it->t_lwp = t->t_lwp; 3180Sstevel@tonic-gate 3190Sstevel@tonic-gate /* 3200Sstevel@tonic-gate * (threads on the interrupt thread free list could have state 3210Sstevel@tonic-gate * preset to TS_ONPROC, but it helps in debugging if 3220Sstevel@tonic-gate * they're TS_FREE.) 3230Sstevel@tonic-gate */ 3240Sstevel@tonic-gate it->t_state = TS_ONPROC; 3250Sstevel@tonic-gate 3260Sstevel@tonic-gate cpu->cpu_thread = it; /* new curthread on this cpu */ 3270Sstevel@tonic-gate it->t_pil = (uchar_t)pil; 3280Sstevel@tonic-gate it->t_pri = intr_pri + (pri_t)pil; 3290Sstevel@tonic-gate it->t_intr_start = tsc_read(); 3300Sstevel@tonic-gate 3310Sstevel@tonic-gate return (it->t_stk); 3320Sstevel@tonic-gate } 3330Sstevel@tonic-gate 3340Sstevel@tonic-gate 3350Sstevel@tonic-gate #ifdef DEBUG 3360Sstevel@tonic-gate int intr_thread_cnt; 3370Sstevel@tonic-gate #endif 3380Sstevel@tonic-gate 3390Sstevel@tonic-gate /* 3400Sstevel@tonic-gate * Called with interrupts disabled 3410Sstevel@tonic-gate */ 3420Sstevel@tonic-gate void 3430Sstevel@tonic-gate intr_thread_epilog(struct cpu *cpu, uint_t vec, uint_t oldpil) 3440Sstevel@tonic-gate { 3450Sstevel@tonic-gate struct machcpu *mcpu = &cpu->cpu_m; 3460Sstevel@tonic-gate kthread_t *t; 3470Sstevel@tonic-gate kthread_t *it = cpu->cpu_thread; /* curthread */ 3480Sstevel@tonic-gate uint_t pil, basespl; 349590Sesolom hrtime_t intrtime; 3500Sstevel@tonic-gate 3510Sstevel@tonic-gate pil = it->t_pil; 3520Sstevel@tonic-gate cpu->cpu_stats.sys.intr[pil - 1]++; 3530Sstevel@tonic-gate 3540Sstevel@tonic-gate ASSERT(it->t_intr_start != 0); 355590Sesolom intrtime = tsc_read() - it->t_intr_start; 356916Sschwartz mcpu->intrstat[pil][0] += intrtime; 357590Sesolom cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 3580Sstevel@tonic-gate 3590Sstevel@tonic-gate ASSERT(cpu->cpu_intr_actv & (1 << pil)); 3600Sstevel@tonic-gate cpu->cpu_intr_actv &= ~(1 << pil); 3610Sstevel@tonic-gate 3620Sstevel@tonic-gate /* 3630Sstevel@tonic-gate * If there is still an interrupted thread underneath this one 3640Sstevel@tonic-gate * then the interrupt was never blocked and the return is 3650Sstevel@tonic-gate * fairly simple. Otherwise it isn't. 3660Sstevel@tonic-gate */ 3670Sstevel@tonic-gate if ((t = it->t_intr) == NULL) { 3680Sstevel@tonic-gate /* 3690Sstevel@tonic-gate * The interrupted thread is no longer pinned underneath 3700Sstevel@tonic-gate * the interrupt thread. This means the interrupt must 3710Sstevel@tonic-gate * have blocked, and the interrupted thread has been 3720Sstevel@tonic-gate * unpinned, and has probably been running around the 3730Sstevel@tonic-gate * system for a while. 3740Sstevel@tonic-gate * 3750Sstevel@tonic-gate * Since there is no longer a thread under this one, put 3760Sstevel@tonic-gate * this interrupt thread back on the CPU's free list and 3770Sstevel@tonic-gate * resume the idle thread which will dispatch the next 3780Sstevel@tonic-gate * thread to run. 3790Sstevel@tonic-gate */ 3800Sstevel@tonic-gate #ifdef DEBUG 3810Sstevel@tonic-gate intr_thread_cnt++; 3820Sstevel@tonic-gate #endif 3830Sstevel@tonic-gate cpu->cpu_stats.sys.intrblk++; 3840Sstevel@tonic-gate /* 3850Sstevel@tonic-gate * Set CPU's base SPL based on active interrupts bitmask 3860Sstevel@tonic-gate */ 3870Sstevel@tonic-gate set_base_spl(); 3880Sstevel@tonic-gate basespl = cpu->cpu_base_spl; 3890Sstevel@tonic-gate mcpu->mcpu_pri = basespl; 3900Sstevel@tonic-gate (*setlvlx)(basespl, vec); 3910Sstevel@tonic-gate (void) splhigh(); 3920Sstevel@tonic-gate it->t_state = TS_FREE; 3930Sstevel@tonic-gate /* 3940Sstevel@tonic-gate * Return interrupt thread to pool 3950Sstevel@tonic-gate */ 3960Sstevel@tonic-gate it->t_link = cpu->cpu_intr_thread; 3970Sstevel@tonic-gate cpu->cpu_intr_thread = it; 3980Sstevel@tonic-gate swtch(); 3990Sstevel@tonic-gate /*NOTREACHED*/ 4000Sstevel@tonic-gate } 4010Sstevel@tonic-gate 4020Sstevel@tonic-gate /* 4030Sstevel@tonic-gate * Return interrupt thread to the pool 4040Sstevel@tonic-gate */ 4050Sstevel@tonic-gate it->t_link = cpu->cpu_intr_thread; 4060Sstevel@tonic-gate cpu->cpu_intr_thread = it; 4070Sstevel@tonic-gate it->t_state = TS_FREE; 4080Sstevel@tonic-gate 4090Sstevel@tonic-gate basespl = cpu->cpu_base_spl; 4100Sstevel@tonic-gate pil = MAX(oldpil, basespl); 4110Sstevel@tonic-gate mcpu->mcpu_pri = pil; 4120Sstevel@tonic-gate (*setlvlx)(pil, vec); 4130Sstevel@tonic-gate t->t_intr_start = tsc_read(); 4140Sstevel@tonic-gate cpu->cpu_thread = t; 4150Sstevel@tonic-gate } 4160Sstevel@tonic-gate 417916Sschwartz /* 418916Sschwartz * Called with interrupts disabled by an interrupt thread to determine 419916Sschwartz * how much time has elapsed. See interrupt.s:intr_get_time() for detailed 420916Sschwartz * theory of operation. 421916Sschwartz */ 422916Sschwartz uint64_t 423916Sschwartz intr_thread_get_time(struct cpu *cpu) 424916Sschwartz { 425916Sschwartz struct machcpu *mcpu = &cpu->cpu_m; 426916Sschwartz kthread_t *t = cpu->cpu_thread; 427916Sschwartz uint64_t time, delta, ret; 428916Sschwartz uint_t pil = t->t_pil; 429916Sschwartz 430916Sschwartz ASSERT((cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK) == 0); 431916Sschwartz ASSERT(t->t_flag & T_INTR_THREAD); 432916Sschwartz ASSERT(pil != 0); 433916Sschwartz ASSERT(t->t_intr_start != 0); 434916Sschwartz 435916Sschwartz time = tsc_read(); 436916Sschwartz delta = time - t->t_intr_start; 437916Sschwartz t->t_intr_start = time; 438916Sschwartz 439916Sschwartz time = mcpu->intrstat[pil][0] + delta; 440916Sschwartz ret = time - mcpu->intrstat[pil][1]; 441916Sschwartz mcpu->intrstat[pil][0] = time; 442916Sschwartz mcpu->intrstat[pil][1] = time; 443916Sschwartz 444916Sschwartz return (ret); 445916Sschwartz } 446916Sschwartz 4470Sstevel@tonic-gate caddr_t 4480Sstevel@tonic-gate dosoftint_prolog( 4490Sstevel@tonic-gate struct cpu *cpu, 4500Sstevel@tonic-gate caddr_t stackptr, 4510Sstevel@tonic-gate uint32_t st_pending, 4520Sstevel@tonic-gate uint_t oldpil) 4530Sstevel@tonic-gate { 4540Sstevel@tonic-gate kthread_t *t, *volatile it; 4550Sstevel@tonic-gate struct machcpu *mcpu = &cpu->cpu_m; 4560Sstevel@tonic-gate uint_t pil; 4570Sstevel@tonic-gate 4580Sstevel@tonic-gate top: 4590Sstevel@tonic-gate ASSERT(st_pending == mcpu->mcpu_softinfo.st_pending); 4600Sstevel@tonic-gate 4610Sstevel@tonic-gate pil = bsrw_insn((uint16_t)st_pending); 4620Sstevel@tonic-gate if (pil <= oldpil || pil <= cpu->cpu_base_spl) 4630Sstevel@tonic-gate return (0); 4640Sstevel@tonic-gate 4650Sstevel@tonic-gate /* 4660Sstevel@tonic-gate * XX64 Sigh. 4670Sstevel@tonic-gate * 4680Sstevel@tonic-gate * This is a transliteration of the i386 assembler code for 4690Sstevel@tonic-gate * soft interrupts. One question is "why does this need 4700Sstevel@tonic-gate * to be atomic?" One possible race is -other- processors 4710Sstevel@tonic-gate * posting soft interrupts to us in set_pending() i.e. the 4720Sstevel@tonic-gate * CPU might get preempted just after the address computation, 4730Sstevel@tonic-gate * but just before the atomic transaction, so another CPU would 4740Sstevel@tonic-gate * actually set the original CPU's st_pending bit. However, 4750Sstevel@tonic-gate * it looks like it would be simpler to disable preemption there. 4760Sstevel@tonic-gate * Are there other races for which preemption control doesn't work? 4770Sstevel@tonic-gate * 4780Sstevel@tonic-gate * The i386 assembler version -also- checks to see if the bit 4790Sstevel@tonic-gate * being cleared was actually set; if it wasn't, it rechecks 4800Sstevel@tonic-gate * for more. This seems a bit strange, as the only code that 4810Sstevel@tonic-gate * ever clears the bit is -this- code running with interrupts 4820Sstevel@tonic-gate * disabled on -this- CPU. This code would probably be cheaper: 4830Sstevel@tonic-gate * 4840Sstevel@tonic-gate * atomic_and_32((uint32_t *)&mcpu->mcpu_softinfo.st_pending, 4850Sstevel@tonic-gate * ~(1 << pil)); 4860Sstevel@tonic-gate * 4870Sstevel@tonic-gate * and t->t_preempt--/++ around set_pending() even cheaper, 4880Sstevel@tonic-gate * but at this point, correctness is critical, so we slavishly 4890Sstevel@tonic-gate * emulate the i386 port. 4900Sstevel@tonic-gate */ 4910Sstevel@tonic-gate if (atomic_btr32((uint32_t *)&mcpu->mcpu_softinfo.st_pending, pil) 4920Sstevel@tonic-gate == 0) { 4930Sstevel@tonic-gate st_pending = mcpu->mcpu_softinfo.st_pending; 4940Sstevel@tonic-gate goto top; 4950Sstevel@tonic-gate } 4960Sstevel@tonic-gate 4970Sstevel@tonic-gate mcpu->mcpu_pri = pil; 4980Sstevel@tonic-gate (*setspl)(pil); 4990Sstevel@tonic-gate 5000Sstevel@tonic-gate /* 5010Sstevel@tonic-gate * Get set to run interrupt thread. 5020Sstevel@tonic-gate * There should always be an interrupt thread since we 5030Sstevel@tonic-gate * allocate one for each level on the CPU. 5040Sstevel@tonic-gate */ 5050Sstevel@tonic-gate it = cpu->cpu_intr_thread; 5060Sstevel@tonic-gate cpu->cpu_intr_thread = it->t_link; 5070Sstevel@tonic-gate 508989Sesolom /* t_intr_start could be zero due to cpu_intr_swtch_enter. */ 509989Sesolom t = cpu->cpu_thread; 510989Sesolom if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) { 511989Sesolom hrtime_t intrtime = tsc_read() - t->t_intr_start; 512989Sesolom mcpu->intrstat[pil][0] += intrtime; 513989Sesolom cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 514989Sesolom t->t_intr_start = 0; 515989Sesolom } 516989Sesolom 5170Sstevel@tonic-gate /* 5180Sstevel@tonic-gate * Note that the code in kcpc_overflow_intr -relies- on the 5190Sstevel@tonic-gate * ordering of events here - in particular that t->t_lwp of 5200Sstevel@tonic-gate * the interrupt thread is set to the pinned thread *before* 521989Sesolom * curthread is changed. 5220Sstevel@tonic-gate */ 5230Sstevel@tonic-gate it->t_lwp = t->t_lwp; 5240Sstevel@tonic-gate it->t_state = TS_ONPROC; 5250Sstevel@tonic-gate 5260Sstevel@tonic-gate /* 5270Sstevel@tonic-gate * Push interrupted thread onto list from new thread. 5280Sstevel@tonic-gate * Set the new thread as the current one. 5290Sstevel@tonic-gate * Set interrupted thread's T_SP because if it is the idle thread, 5300Sstevel@tonic-gate * resume() may use that stack between threads. 5310Sstevel@tonic-gate */ 5320Sstevel@tonic-gate 5330Sstevel@tonic-gate ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr); 5340Sstevel@tonic-gate t->t_sp = (uintptr_t)stackptr; 5350Sstevel@tonic-gate 5360Sstevel@tonic-gate it->t_intr = t; 5370Sstevel@tonic-gate cpu->cpu_thread = it; 5380Sstevel@tonic-gate 5390Sstevel@tonic-gate /* 5400Sstevel@tonic-gate * Set bit for this pil in CPU's interrupt active bitmask. 5410Sstevel@tonic-gate */ 5420Sstevel@tonic-gate ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0); 5430Sstevel@tonic-gate cpu->cpu_intr_actv |= (1 << pil); 5440Sstevel@tonic-gate 5450Sstevel@tonic-gate /* 5460Sstevel@tonic-gate * Initialize thread priority level from intr_pri 5470Sstevel@tonic-gate */ 5480Sstevel@tonic-gate it->t_pil = (uchar_t)pil; 5490Sstevel@tonic-gate it->t_pri = (pri_t)pil + intr_pri; 5500Sstevel@tonic-gate it->t_intr_start = tsc_read(); 5510Sstevel@tonic-gate 5520Sstevel@tonic-gate return (it->t_stk); 5530Sstevel@tonic-gate } 5540Sstevel@tonic-gate 5550Sstevel@tonic-gate void 5560Sstevel@tonic-gate dosoftint_epilog(struct cpu *cpu, uint_t oldpil) 5570Sstevel@tonic-gate { 5580Sstevel@tonic-gate struct machcpu *mcpu = &cpu->cpu_m; 5590Sstevel@tonic-gate kthread_t *t, *it; 5600Sstevel@tonic-gate uint_t pil, basespl; 561590Sesolom hrtime_t intrtime; 5620Sstevel@tonic-gate 5630Sstevel@tonic-gate it = cpu->cpu_thread; 5640Sstevel@tonic-gate pil = it->t_pil; 5650Sstevel@tonic-gate 5660Sstevel@tonic-gate cpu->cpu_stats.sys.intr[pil - 1]++; 5670Sstevel@tonic-gate 5680Sstevel@tonic-gate ASSERT(cpu->cpu_intr_actv & (1 << pil)); 5690Sstevel@tonic-gate cpu->cpu_intr_actv &= ~(1 << pil); 570590Sesolom intrtime = tsc_read() - it->t_intr_start; 571916Sschwartz mcpu->intrstat[pil][0] += intrtime; 572590Sesolom cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; 5730Sstevel@tonic-gate 5740Sstevel@tonic-gate /* 5750Sstevel@tonic-gate * If there is still an interrupted thread underneath this one 5760Sstevel@tonic-gate * then the interrupt was never blocked and the return is 5770Sstevel@tonic-gate * fairly simple. Otherwise it isn't. 5780Sstevel@tonic-gate */ 5790Sstevel@tonic-gate if ((t = it->t_intr) == NULL) { 5800Sstevel@tonic-gate /* 5810Sstevel@tonic-gate * Put thread back on the interrupt thread list. 5820Sstevel@tonic-gate * This was an interrupt thread, so set CPU's base SPL. 5830Sstevel@tonic-gate */ 5840Sstevel@tonic-gate set_base_spl(); 5850Sstevel@tonic-gate it->t_state = TS_FREE; 5860Sstevel@tonic-gate it->t_link = cpu->cpu_intr_thread; 5870Sstevel@tonic-gate cpu->cpu_intr_thread = it; 5880Sstevel@tonic-gate (void) splhigh(); 5890Sstevel@tonic-gate swtch(); 5900Sstevel@tonic-gate /*NOTREACHED*/ 5910Sstevel@tonic-gate } 5920Sstevel@tonic-gate it->t_link = cpu->cpu_intr_thread; 5930Sstevel@tonic-gate cpu->cpu_intr_thread = it; 5940Sstevel@tonic-gate it->t_state = TS_FREE; 5950Sstevel@tonic-gate cpu->cpu_thread = t; 5960Sstevel@tonic-gate if (t->t_flag & T_INTR_THREAD) 5970Sstevel@tonic-gate t->t_intr_start = tsc_read(); 5980Sstevel@tonic-gate basespl = cpu->cpu_base_spl; 5990Sstevel@tonic-gate pil = MAX(oldpil, basespl); 6000Sstevel@tonic-gate mcpu->mcpu_pri = pil; 6010Sstevel@tonic-gate (*setspl)(pil); 6020Sstevel@tonic-gate } 6030Sstevel@tonic-gate 6040Sstevel@tonic-gate /* 6050Sstevel@tonic-gate * Make the interrupted thread 'to' be runnable. 6060Sstevel@tonic-gate * 6070Sstevel@tonic-gate * Since t->t_sp has already been saved, t->t_pc is all 6080Sstevel@tonic-gate * that needs to be set in this function. 6090Sstevel@tonic-gate * 6100Sstevel@tonic-gate * Returns the interrupt level of the interrupt thread. 6110Sstevel@tonic-gate */ 6120Sstevel@tonic-gate int 6130Sstevel@tonic-gate intr_passivate( 6140Sstevel@tonic-gate kthread_t *it, /* interrupt thread */ 6150Sstevel@tonic-gate kthread_t *t) /* interrupted thread */ 6160Sstevel@tonic-gate { 6170Sstevel@tonic-gate extern void _sys_rtt(); 6180Sstevel@tonic-gate 6190Sstevel@tonic-gate ASSERT(it->t_flag & T_INTR_THREAD); 6200Sstevel@tonic-gate ASSERT(SA(t->t_sp) == t->t_sp); 6210Sstevel@tonic-gate 6220Sstevel@tonic-gate t->t_pc = (uintptr_t)_sys_rtt; 6230Sstevel@tonic-gate return (it->t_pil); 6240Sstevel@tonic-gate } 6250Sstevel@tonic-gate 6260Sstevel@tonic-gate #endif /* __amd64 */ 6270Sstevel@tonic-gate 6280Sstevel@tonic-gate /* 6290Sstevel@tonic-gate * Create interrupt kstats for this CPU. 6300Sstevel@tonic-gate */ 6310Sstevel@tonic-gate void 6320Sstevel@tonic-gate cpu_create_intrstat(cpu_t *cp) 6330Sstevel@tonic-gate { 6340Sstevel@tonic-gate int i; 6350Sstevel@tonic-gate kstat_t *intr_ksp; 6360Sstevel@tonic-gate kstat_named_t *knp; 6370Sstevel@tonic-gate char name[KSTAT_STRLEN]; 6380Sstevel@tonic-gate zoneid_t zoneid; 6390Sstevel@tonic-gate 6400Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 6410Sstevel@tonic-gate 6420Sstevel@tonic-gate if (pool_pset_enabled()) 6430Sstevel@tonic-gate zoneid = GLOBAL_ZONEID; 6440Sstevel@tonic-gate else 6450Sstevel@tonic-gate zoneid = ALL_ZONES; 6460Sstevel@tonic-gate 6470Sstevel@tonic-gate intr_ksp = kstat_create_zone("cpu", cp->cpu_id, "intrstat", "misc", 6480Sstevel@tonic-gate KSTAT_TYPE_NAMED, PIL_MAX * 2, NULL, zoneid); 6490Sstevel@tonic-gate 6500Sstevel@tonic-gate /* 6510Sstevel@tonic-gate * Initialize each PIL's named kstat 6520Sstevel@tonic-gate */ 6530Sstevel@tonic-gate if (intr_ksp != NULL) { 6540Sstevel@tonic-gate intr_ksp->ks_update = cpu_kstat_intrstat_update; 6550Sstevel@tonic-gate knp = (kstat_named_t *)intr_ksp->ks_data; 6560Sstevel@tonic-gate intr_ksp->ks_private = cp; 6570Sstevel@tonic-gate for (i = 0; i < PIL_MAX; i++) { 6580Sstevel@tonic-gate (void) snprintf(name, KSTAT_STRLEN, "level-%d-time", 6590Sstevel@tonic-gate i + 1); 6600Sstevel@tonic-gate kstat_named_init(&knp[i * 2], name, KSTAT_DATA_UINT64); 6610Sstevel@tonic-gate (void) snprintf(name, KSTAT_STRLEN, "level-%d-count", 6620Sstevel@tonic-gate i + 1); 6630Sstevel@tonic-gate kstat_named_init(&knp[(i * 2) + 1], name, 6640Sstevel@tonic-gate KSTAT_DATA_UINT64); 6650Sstevel@tonic-gate } 6660Sstevel@tonic-gate kstat_install(intr_ksp); 6670Sstevel@tonic-gate } 6680Sstevel@tonic-gate } 6690Sstevel@tonic-gate 6700Sstevel@tonic-gate /* 6710Sstevel@tonic-gate * Delete interrupt kstats for this CPU. 6720Sstevel@tonic-gate */ 6730Sstevel@tonic-gate void 6740Sstevel@tonic-gate cpu_delete_intrstat(cpu_t *cp) 6750Sstevel@tonic-gate { 6760Sstevel@tonic-gate kstat_delete_byname_zone("cpu", cp->cpu_id, "intrstat", ALL_ZONES); 6770Sstevel@tonic-gate } 6780Sstevel@tonic-gate 6790Sstevel@tonic-gate /* 6800Sstevel@tonic-gate * Convert interrupt statistics from CPU ticks to nanoseconds and 6810Sstevel@tonic-gate * update kstat. 6820Sstevel@tonic-gate */ 6830Sstevel@tonic-gate int 6840Sstevel@tonic-gate cpu_kstat_intrstat_update(kstat_t *ksp, int rw) 6850Sstevel@tonic-gate { 6860Sstevel@tonic-gate kstat_named_t *knp = ksp->ks_data; 6870Sstevel@tonic-gate cpu_t *cpup = (cpu_t *)ksp->ks_private; 6880Sstevel@tonic-gate int i; 6890Sstevel@tonic-gate hrtime_t hrt; 6900Sstevel@tonic-gate 6910Sstevel@tonic-gate if (rw == KSTAT_WRITE) 6920Sstevel@tonic-gate return (EACCES); 6930Sstevel@tonic-gate 6940Sstevel@tonic-gate for (i = 0; i < PIL_MAX; i++) { 695916Sschwartz hrt = (hrtime_t)cpup->cpu_m.intrstat[i + 1][0]; 6960Sstevel@tonic-gate tsc_scalehrtime(&hrt); 6970Sstevel@tonic-gate knp[i * 2].value.ui64 = (uint64_t)hrt; 6980Sstevel@tonic-gate knp[(i * 2) + 1].value.ui64 = cpup->cpu_stats.sys.intr[i]; 6990Sstevel@tonic-gate } 7000Sstevel@tonic-gate 7010Sstevel@tonic-gate return (0); 7020Sstevel@tonic-gate } 7030Sstevel@tonic-gate 7040Sstevel@tonic-gate /* 7050Sstevel@tonic-gate * An interrupt thread is ending a time slice, so compute the interval it 7060Sstevel@tonic-gate * ran for and update the statistic for its PIL. 7070Sstevel@tonic-gate */ 7080Sstevel@tonic-gate void 7090Sstevel@tonic-gate cpu_intr_swtch_enter(kthread_id_t t) 7100Sstevel@tonic-gate { 7110Sstevel@tonic-gate uint64_t interval; 7120Sstevel@tonic-gate uint64_t start; 713590Sesolom cpu_t *cpu; 7140Sstevel@tonic-gate 7150Sstevel@tonic-gate ASSERT((t->t_flag & T_INTR_THREAD) != 0); 7160Sstevel@tonic-gate ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL); 7170Sstevel@tonic-gate 7180Sstevel@tonic-gate /* 7190Sstevel@tonic-gate * We could be here with a zero timestamp. This could happen if: 7200Sstevel@tonic-gate * an interrupt thread which no longer has a pinned thread underneath 7210Sstevel@tonic-gate * it (i.e. it blocked at some point in its past) has finished running 7220Sstevel@tonic-gate * its handler. intr_thread() updated the interrupt statistic for its 7230Sstevel@tonic-gate * PIL and zeroed its timestamp. Since there was no pinned thread to 7240Sstevel@tonic-gate * return to, swtch() gets called and we end up here. 725590Sesolom * 726590Sesolom * Note that we use atomic ops below (cas64 and atomic_add_64), which 727590Sesolom * we don't use in the functions above, because we're not called 728590Sesolom * with interrupts blocked, but the epilog/prolog functions are. 7290Sstevel@tonic-gate */ 7300Sstevel@tonic-gate if (t->t_intr_start) { 7310Sstevel@tonic-gate do { 7320Sstevel@tonic-gate start = t->t_intr_start; 7330Sstevel@tonic-gate interval = tsc_read() - start; 7340Sstevel@tonic-gate } while (cas64(&t->t_intr_start, start, 0) != start); 735590Sesolom cpu = CPU; 736916Sschwartz cpu->cpu_m.intrstat[t->t_pil][0] += interval; 737590Sesolom 738590Sesolom atomic_add_64((uint64_t *)&cpu->cpu_intracct[cpu->cpu_mstate], 739590Sesolom interval); 7400Sstevel@tonic-gate } else 7410Sstevel@tonic-gate ASSERT(t->t_intr == NULL); 7420Sstevel@tonic-gate } 7430Sstevel@tonic-gate 7440Sstevel@tonic-gate /* 7450Sstevel@tonic-gate * An interrupt thread is returning from swtch(). Place a starting timestamp 7460Sstevel@tonic-gate * in its thread structure. 7470Sstevel@tonic-gate */ 7480Sstevel@tonic-gate void 7490Sstevel@tonic-gate cpu_intr_swtch_exit(kthread_id_t t) 7500Sstevel@tonic-gate { 7510Sstevel@tonic-gate uint64_t ts; 7520Sstevel@tonic-gate 7530Sstevel@tonic-gate ASSERT((t->t_flag & T_INTR_THREAD) != 0); 7540Sstevel@tonic-gate ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL); 7550Sstevel@tonic-gate 7560Sstevel@tonic-gate do { 7570Sstevel@tonic-gate ts = t->t_intr_start; 7580Sstevel@tonic-gate } while (cas64(&t->t_intr_start, ts, tsc_read()) != ts); 7590Sstevel@tonic-gate } 760