1*0Sstevel@tonic-gate /* 2*0Sstevel@tonic-gate * CDDL HEADER START 3*0Sstevel@tonic-gate * 4*0Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*0Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*0Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*0Sstevel@tonic-gate * with the License. 8*0Sstevel@tonic-gate * 9*0Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*0Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*0Sstevel@tonic-gate * See the License for the specific language governing permissions 12*0Sstevel@tonic-gate * and limitations under the License. 13*0Sstevel@tonic-gate * 14*0Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*0Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*0Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*0Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*0Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*0Sstevel@tonic-gate * 20*0Sstevel@tonic-gate * CDDL HEADER END 21*0Sstevel@tonic-gate */ 22*0Sstevel@tonic-gate /* 23*0Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24*0Sstevel@tonic-gate * Use is subject to license terms. 25*0Sstevel@tonic-gate */ 26*0Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 27*0Sstevel@tonic-gate 28*0Sstevel@tonic-gate #include <sys/systm.h> 29*0Sstevel@tonic-gate #include <sys/cyclic.h> 30*0Sstevel@tonic-gate #include <sys/cyclic_impl.h> 31*0Sstevel@tonic-gate #include <sys/spl.h> 32*0Sstevel@tonic-gate #include <sys/x_call.h> 33*0Sstevel@tonic-gate #include <sys/kmem.h> 34*0Sstevel@tonic-gate #include <sys/machsystm.h> 35*0Sstevel@tonic-gate #include <sys/smp_impldefs.h> 36*0Sstevel@tonic-gate #include <sys/psm_types.h> 37*0Sstevel@tonic-gate #include <sys/atomic.h> 38*0Sstevel@tonic-gate #include <sys/clock.h> 39*0Sstevel@tonic-gate #include <sys/ddi_impldefs.h> 40*0Sstevel@tonic-gate #include <sys/ddi_intr.h> 41*0Sstevel@tonic-gate 42*0Sstevel@tonic-gate static int cbe_vector; 43*0Sstevel@tonic-gate static int cbe_ticks = 0; 44*0Sstevel@tonic-gate 45*0Sstevel@tonic-gate static cyc_func_t volatile cbe_xcall_func; 46*0Sstevel@tonic-gate static cpu_t *volatile cbe_xcall_cpu; 47*0Sstevel@tonic-gate static void *cbe_xcall_farg; 48*0Sstevel@tonic-gate static cpuset_t cbe_enabled; 49*0Sstevel@tonic-gate 50*0Sstevel@tonic-gate static ddi_softint_hdl_impl_t cbe_low_hdl = 51*0Sstevel@tonic-gate {0, NULL, NULL, 0, 0, NULL, NULL, NULL}; 52*0Sstevel@tonic-gate static ddi_softint_hdl_impl_t cbe_clock_hdl = 53*0Sstevel@tonic-gate {0, NULL, NULL, 0, 0, NULL, NULL, NULL}; 54*0Sstevel@tonic-gate 55*0Sstevel@tonic-gate cyclic_id_t cbe_hres_cyclic; 56*0Sstevel@tonic-gate int cbe_psm_timer_mode = TIMER_ONESHOT; 57*0Sstevel@tonic-gate 58*0Sstevel@tonic-gate void cbe_hres_tick(void); 59*0Sstevel@tonic-gate 60*0Sstevel@tonic-gate int 61*0Sstevel@tonic-gate cbe_softclock(void) 62*0Sstevel@tonic-gate { 63*0Sstevel@tonic-gate cyclic_softint(CPU, CY_LOCK_LEVEL); 64*0Sstevel@tonic-gate return (1); 65*0Sstevel@tonic-gate } 66*0Sstevel@tonic-gate 67*0Sstevel@tonic-gate int 68*0Sstevel@tonic-gate cbe_low_level(void) 69*0Sstevel@tonic-gate { 70*0Sstevel@tonic-gate cpu_t *cpu = CPU; 71*0Sstevel@tonic-gate 72*0Sstevel@tonic-gate cyclic_softint(cpu, CY_LOW_LEVEL); 73*0Sstevel@tonic-gate return (1); 74*0Sstevel@tonic-gate } 75*0Sstevel@tonic-gate 76*0Sstevel@tonic-gate /* 77*0Sstevel@tonic-gate * We can be in cbe_fire() either due to a cyclic-induced cross call, or due 78*0Sstevel@tonic-gate * to the timer firing at level-14. Because cyclic_fire() can tolerate 79*0Sstevel@tonic-gate * spurious calls, it would not matter if we called cyclic_fire() in both 80*0Sstevel@tonic-gate * cases. 81*0Sstevel@tonic-gate * 82*0Sstevel@tonic-gate */ 83*0Sstevel@tonic-gate int 84*0Sstevel@tonic-gate cbe_fire(void) 85*0Sstevel@tonic-gate { 86*0Sstevel@tonic-gate cpu_t *cpu = CPU; 87*0Sstevel@tonic-gate processorid_t me = cpu->cpu_id, i; 88*0Sstevel@tonic-gate int cross_call = (cbe_xcall_func != NULL && cbe_xcall_cpu == cpu); 89*0Sstevel@tonic-gate 90*0Sstevel@tonic-gate cyclic_fire(cpu); 91*0Sstevel@tonic-gate 92*0Sstevel@tonic-gate if (cbe_psm_timer_mode != TIMER_ONESHOT && me == 0 && !cross_call) { 93*0Sstevel@tonic-gate for (i = 1; i < NCPU; i++) { 94*0Sstevel@tonic-gate if (CPU_IN_SET(cbe_enabled, i)) 95*0Sstevel@tonic-gate send_dirint(i, CBE_HIGH_PIL); 96*0Sstevel@tonic-gate } 97*0Sstevel@tonic-gate } 98*0Sstevel@tonic-gate 99*0Sstevel@tonic-gate if (cross_call) { 100*0Sstevel@tonic-gate ASSERT(cbe_xcall_func != NULL && cbe_xcall_cpu == cpu); 101*0Sstevel@tonic-gate (*cbe_xcall_func)(cbe_xcall_farg); 102*0Sstevel@tonic-gate cbe_xcall_func = NULL; 103*0Sstevel@tonic-gate cbe_xcall_cpu = NULL; 104*0Sstevel@tonic-gate } 105*0Sstevel@tonic-gate 106*0Sstevel@tonic-gate return (1); 107*0Sstevel@tonic-gate } 108*0Sstevel@tonic-gate 109*0Sstevel@tonic-gate /*ARGSUSED*/ 110*0Sstevel@tonic-gate void 111*0Sstevel@tonic-gate cbe_softint(void *arg, cyc_level_t level) 112*0Sstevel@tonic-gate { 113*0Sstevel@tonic-gate switch (level) { 114*0Sstevel@tonic-gate case CY_LOW_LEVEL: 115*0Sstevel@tonic-gate cbe_low_hdl.ih_pending = 1; 116*0Sstevel@tonic-gate (*setsoftint)(CBE_LOW_PIL); 117*0Sstevel@tonic-gate break; 118*0Sstevel@tonic-gate case CY_LOCK_LEVEL: 119*0Sstevel@tonic-gate cbe_clock_hdl.ih_pending = 1; 120*0Sstevel@tonic-gate (*setsoftint)(CBE_LOCK_PIL); 121*0Sstevel@tonic-gate break; 122*0Sstevel@tonic-gate default: 123*0Sstevel@tonic-gate panic("cbe_softint: unexpected soft level %d", level); 124*0Sstevel@tonic-gate } 125*0Sstevel@tonic-gate } 126*0Sstevel@tonic-gate 127*0Sstevel@tonic-gate /*ARGSUSED*/ 128*0Sstevel@tonic-gate void 129*0Sstevel@tonic-gate cbe_reprogram(void *arg, hrtime_t time) 130*0Sstevel@tonic-gate { 131*0Sstevel@tonic-gate if (cbe_psm_timer_mode == TIMER_ONESHOT) 132*0Sstevel@tonic-gate (*psm_timer_reprogram)(time); 133*0Sstevel@tonic-gate } 134*0Sstevel@tonic-gate 135*0Sstevel@tonic-gate /*ARGSUSED*/ 136*0Sstevel@tonic-gate cyc_cookie_t 137*0Sstevel@tonic-gate cbe_set_level(void *arg, cyc_level_t level) 138*0Sstevel@tonic-gate { 139*0Sstevel@tonic-gate int ipl; 140*0Sstevel@tonic-gate 141*0Sstevel@tonic-gate switch (level) { 142*0Sstevel@tonic-gate case CY_LOW_LEVEL: 143*0Sstevel@tonic-gate ipl = CBE_LOW_PIL; 144*0Sstevel@tonic-gate break; 145*0Sstevel@tonic-gate case CY_LOCK_LEVEL: 146*0Sstevel@tonic-gate ipl = CBE_LOCK_PIL; 147*0Sstevel@tonic-gate break; 148*0Sstevel@tonic-gate case CY_HIGH_LEVEL: 149*0Sstevel@tonic-gate ipl = CBE_HIGH_PIL; 150*0Sstevel@tonic-gate break; 151*0Sstevel@tonic-gate default: 152*0Sstevel@tonic-gate panic("cbe_set_level: unexpected level %d", level); 153*0Sstevel@tonic-gate } 154*0Sstevel@tonic-gate 155*0Sstevel@tonic-gate return (splr(ipltospl(ipl))); 156*0Sstevel@tonic-gate } 157*0Sstevel@tonic-gate 158*0Sstevel@tonic-gate /*ARGSUSED*/ 159*0Sstevel@tonic-gate void 160*0Sstevel@tonic-gate cbe_restore_level(void *arg, cyc_cookie_t cookie) 161*0Sstevel@tonic-gate { 162*0Sstevel@tonic-gate splx(cookie); 163*0Sstevel@tonic-gate } 164*0Sstevel@tonic-gate 165*0Sstevel@tonic-gate /*ARGSUSED*/ 166*0Sstevel@tonic-gate void 167*0Sstevel@tonic-gate cbe_xcall(void *arg, cpu_t *dest, cyc_func_t func, void *farg) 168*0Sstevel@tonic-gate { 169*0Sstevel@tonic-gate kpreempt_disable(); 170*0Sstevel@tonic-gate 171*0Sstevel@tonic-gate if (dest == CPU) { 172*0Sstevel@tonic-gate (*func)(farg); 173*0Sstevel@tonic-gate kpreempt_enable(); 174*0Sstevel@tonic-gate return; 175*0Sstevel@tonic-gate } 176*0Sstevel@tonic-gate 177*0Sstevel@tonic-gate ASSERT(cbe_xcall_func == NULL); 178*0Sstevel@tonic-gate 179*0Sstevel@tonic-gate cbe_xcall_farg = farg; 180*0Sstevel@tonic-gate membar_producer(); 181*0Sstevel@tonic-gate cbe_xcall_cpu = dest; 182*0Sstevel@tonic-gate cbe_xcall_func = func; 183*0Sstevel@tonic-gate 184*0Sstevel@tonic-gate send_dirint(dest->cpu_id, CBE_HIGH_PIL); 185*0Sstevel@tonic-gate 186*0Sstevel@tonic-gate while (cbe_xcall_func != NULL || cbe_xcall_cpu != NULL) 187*0Sstevel@tonic-gate continue; 188*0Sstevel@tonic-gate 189*0Sstevel@tonic-gate kpreempt_enable(); 190*0Sstevel@tonic-gate 191*0Sstevel@tonic-gate ASSERT(cbe_xcall_func == NULL && cbe_xcall_cpu == NULL); 192*0Sstevel@tonic-gate } 193*0Sstevel@tonic-gate 194*0Sstevel@tonic-gate void * 195*0Sstevel@tonic-gate cbe_configure(cpu_t *cpu) 196*0Sstevel@tonic-gate { 197*0Sstevel@tonic-gate return (cpu); 198*0Sstevel@tonic-gate } 199*0Sstevel@tonic-gate 200*0Sstevel@tonic-gate void 201*0Sstevel@tonic-gate cbe_enable(void *arg) 202*0Sstevel@tonic-gate { 203*0Sstevel@tonic-gate processorid_t me = ((cpu_t *)arg)->cpu_id; 204*0Sstevel@tonic-gate 205*0Sstevel@tonic-gate if ((cbe_psm_timer_mode != TIMER_ONESHOT) && (me == 0)) 206*0Sstevel@tonic-gate return; 207*0Sstevel@tonic-gate 208*0Sstevel@tonic-gate ASSERT(!CPU_IN_SET(cbe_enabled, me)); 209*0Sstevel@tonic-gate CPUSET_ADD(cbe_enabled, me); 210*0Sstevel@tonic-gate if (cbe_psm_timer_mode == TIMER_ONESHOT) 211*0Sstevel@tonic-gate (*psm_timer_enable)(); 212*0Sstevel@tonic-gate } 213*0Sstevel@tonic-gate 214*0Sstevel@tonic-gate void 215*0Sstevel@tonic-gate cbe_disable(void *arg) 216*0Sstevel@tonic-gate { 217*0Sstevel@tonic-gate processorid_t me = ((cpu_t *)arg)->cpu_id; 218*0Sstevel@tonic-gate 219*0Sstevel@tonic-gate if (me == 0) { 220*0Sstevel@tonic-gate /* 221*0Sstevel@tonic-gate * If this is the boot CPU, we'll quietly refuse to disable 222*0Sstevel@tonic-gate * our clock interrupt. 223*0Sstevel@tonic-gate */ 224*0Sstevel@tonic-gate return; 225*0Sstevel@tonic-gate } 226*0Sstevel@tonic-gate 227*0Sstevel@tonic-gate ASSERT(CPU_IN_SET(cbe_enabled, me)); 228*0Sstevel@tonic-gate CPUSET_DEL(cbe_enabled, me); 229*0Sstevel@tonic-gate if (cbe_psm_timer_mode == TIMER_ONESHOT) 230*0Sstevel@tonic-gate (*psm_timer_disable)(); 231*0Sstevel@tonic-gate } 232*0Sstevel@tonic-gate 233*0Sstevel@tonic-gate /* 234*0Sstevel@tonic-gate * Called only on CPU 0. This is done since TSCs can have deltas between 235*0Sstevel@tonic-gate * different cpus see tsc_tick() 236*0Sstevel@tonic-gate */ 237*0Sstevel@tonic-gate void 238*0Sstevel@tonic-gate cbe_hres_tick(void) 239*0Sstevel@tonic-gate { 240*0Sstevel@tonic-gate int s; 241*0Sstevel@tonic-gate 242*0Sstevel@tonic-gate dtrace_hres_tick(); 243*0Sstevel@tonic-gate 244*0Sstevel@tonic-gate /* 245*0Sstevel@tonic-gate * Because hres_tick effectively locks hres_lock, we must be at the 246*0Sstevel@tonic-gate * same PIL as that used for CLOCK_LOCK. 247*0Sstevel@tonic-gate */ 248*0Sstevel@tonic-gate s = splr(ipltospl(XC_HI_PIL)); 249*0Sstevel@tonic-gate hres_tick(); 250*0Sstevel@tonic-gate splx(s); 251*0Sstevel@tonic-gate 252*0Sstevel@tonic-gate if ((cbe_ticks % hz) == 0) 253*0Sstevel@tonic-gate (*hrtime_tick)(); 254*0Sstevel@tonic-gate 255*0Sstevel@tonic-gate cbe_ticks++; 256*0Sstevel@tonic-gate 257*0Sstevel@tonic-gate } 258*0Sstevel@tonic-gate 259*0Sstevel@tonic-gate void 260*0Sstevel@tonic-gate cbe_init(void) 261*0Sstevel@tonic-gate { 262*0Sstevel@tonic-gate cyc_backend_t cbe = { 263*0Sstevel@tonic-gate cbe_configure, /* cyb_configure */ 264*0Sstevel@tonic-gate NULL, /* cyb_unconfigure */ 265*0Sstevel@tonic-gate cbe_enable, /* cyb_enable */ 266*0Sstevel@tonic-gate cbe_disable, /* cyb_disable */ 267*0Sstevel@tonic-gate cbe_reprogram, /* cyb_reprogram */ 268*0Sstevel@tonic-gate cbe_softint, /* cyb_softint */ 269*0Sstevel@tonic-gate cbe_set_level, /* cyb_set_level */ 270*0Sstevel@tonic-gate cbe_restore_level, /* cyb_restore_level */ 271*0Sstevel@tonic-gate cbe_xcall, /* cyb_xcall */ 272*0Sstevel@tonic-gate NULL, /* cyb_suspend */ 273*0Sstevel@tonic-gate NULL /* cyb_resume */ 274*0Sstevel@tonic-gate }; 275*0Sstevel@tonic-gate hrtime_t resolution; 276*0Sstevel@tonic-gate cyc_handler_t hdlr; 277*0Sstevel@tonic-gate cyc_time_t when; 278*0Sstevel@tonic-gate 279*0Sstevel@tonic-gate cbe_vector = (*psm_get_clockirq)(CBE_HIGH_PIL); 280*0Sstevel@tonic-gate 281*0Sstevel@tonic-gate CPUSET_ZERO(cbe_enabled); 282*0Sstevel@tonic-gate 283*0Sstevel@tonic-gate resolution = (*clkinitf)(TIMER_ONESHOT, &cbe_psm_timer_mode); 284*0Sstevel@tonic-gate 285*0Sstevel@tonic-gate mutex_enter(&cpu_lock); 286*0Sstevel@tonic-gate cyclic_init(&cbe, resolution); 287*0Sstevel@tonic-gate mutex_exit(&cpu_lock); 288*0Sstevel@tonic-gate 289*0Sstevel@tonic-gate (void) add_avintr(NULL, CBE_HIGH_PIL, (avfunc)cbe_fire, 290*0Sstevel@tonic-gate "cbe_fire_master", cbe_vector, 0, NULL, NULL); 291*0Sstevel@tonic-gate 292*0Sstevel@tonic-gate if (psm_get_ipivect != NULL) { 293*0Sstevel@tonic-gate (void) add_avintr(NULL, CBE_HIGH_PIL, (avfunc)cbe_fire, 294*0Sstevel@tonic-gate "cbe_fire_slave", 295*0Sstevel@tonic-gate (*psm_get_ipivect)(CBE_HIGH_PIL, PSM_INTR_IPI_HI), 296*0Sstevel@tonic-gate 0, NULL, NULL); 297*0Sstevel@tonic-gate } 298*0Sstevel@tonic-gate 299*0Sstevel@tonic-gate (void) add_avsoftintr((void *)&cbe_clock_hdl, CBE_LOCK_PIL, 300*0Sstevel@tonic-gate (avfunc)cbe_softclock, "softclock", NULL, NULL); 301*0Sstevel@tonic-gate 302*0Sstevel@tonic-gate (void) add_avsoftintr((void *)&cbe_low_hdl, CBE_LOW_PIL, 303*0Sstevel@tonic-gate (avfunc)cbe_low_level, "low level", NULL, NULL); 304*0Sstevel@tonic-gate 305*0Sstevel@tonic-gate mutex_enter(&cpu_lock); 306*0Sstevel@tonic-gate 307*0Sstevel@tonic-gate hdlr.cyh_level = CY_HIGH_LEVEL; 308*0Sstevel@tonic-gate hdlr.cyh_func = (cyc_func_t)cbe_hres_tick; 309*0Sstevel@tonic-gate hdlr.cyh_arg = NULL; 310*0Sstevel@tonic-gate 311*0Sstevel@tonic-gate when.cyt_when = 0; 312*0Sstevel@tonic-gate when.cyt_interval = nsec_per_tick; 313*0Sstevel@tonic-gate 314*0Sstevel@tonic-gate cbe_hres_cyclic = cyclic_add(&hdlr, &when); 315*0Sstevel@tonic-gate 316*0Sstevel@tonic-gate /* bind to cpu 0, which is also the boot cpu */ 317*0Sstevel@tonic-gate cyclic_bind(cbe_hres_cyclic, CPU, NULL); 318*0Sstevel@tonic-gate 319*0Sstevel@tonic-gate if (psm_post_cyclic_setup != NULL) 320*0Sstevel@tonic-gate (*psm_post_cyclic_setup)(NULL); 321*0Sstevel@tonic-gate 322*0Sstevel@tonic-gate mutex_exit(&cpu_lock); 323*0Sstevel@tonic-gate 324*0Sstevel@tonic-gate } 325