1*0Sstevel@tonic-gate /* 2*0Sstevel@tonic-gate * CDDL HEADER START 3*0Sstevel@tonic-gate * 4*0Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*0Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*0Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*0Sstevel@tonic-gate * with the License. 8*0Sstevel@tonic-gate * 9*0Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*0Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*0Sstevel@tonic-gate * See the License for the specific language governing permissions 12*0Sstevel@tonic-gate * and limitations under the License. 13*0Sstevel@tonic-gate * 14*0Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*0Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*0Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*0Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*0Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*0Sstevel@tonic-gate * 20*0Sstevel@tonic-gate * CDDL HEADER END 21*0Sstevel@tonic-gate */ 22*0Sstevel@tonic-gate /* 23*0Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24*0Sstevel@tonic-gate * Use is subject to license terms. 25*0Sstevel@tonic-gate */ 26*0Sstevel@tonic-gate 27*0Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 28*0Sstevel@tonic-gate 29*0Sstevel@tonic-gate #include <sys/param.h> 30*0Sstevel@tonic-gate #include <sys/time.h> 31*0Sstevel@tonic-gate #include <sys/systm.h> 32*0Sstevel@tonic-gate #include <sys/cmn_err.h> 33*0Sstevel@tonic-gate #include <sys/debug.h> 34*0Sstevel@tonic-gate #include <sys/clock.h> 35*0Sstevel@tonic-gate #include <sys/x_call.h> 36*0Sstevel@tonic-gate #include <sys/cpuvar.h> 37*0Sstevel@tonic-gate #include <sys/promif.h> 38*0Sstevel@tonic-gate #include <sys/kmem.h> 39*0Sstevel@tonic-gate #include <sys/machsystm.h> 40*0Sstevel@tonic-gate #include <sys/ivintr.h> 41*0Sstevel@tonic-gate #include <sys/cyclic.h> 42*0Sstevel@tonic-gate #include <sys/cyclic_impl.h> 43*0Sstevel@tonic-gate 44*0Sstevel@tonic-gate uint32_t cbe_level14_inum; 45*0Sstevel@tonic-gate cyclic_id_t cbe_hres_cyclic; 46*0Sstevel@tonic-gate 47*0Sstevel@tonic-gate static hrtime_t cbe_hrtime_max; 48*0Sstevel@tonic-gate static hrtime_t cbe_suspend_delta = 0; 49*0Sstevel@tonic-gate static hrtime_t cbe_suspend_time = 0; 50*0Sstevel@tonic-gate 51*0Sstevel@tonic-gate static uint64_t 52*0Sstevel@tonic-gate hrtime2tick(hrtime_t ts) 53*0Sstevel@tonic-gate { 54*0Sstevel@tonic-gate hrtime_t q = ts / NANOSEC; 55*0Sstevel@tonic-gate hrtime_t r = ts - (q * NANOSEC); 56*0Sstevel@tonic-gate 57*0Sstevel@tonic-gate return (q * sys_tick_freq + ((r * sys_tick_freq) / NANOSEC)); 58*0Sstevel@tonic-gate } 59*0Sstevel@tonic-gate 60*0Sstevel@tonic-gate uint64_t 61*0Sstevel@tonic-gate unscalehrtime(hrtime_t ts) 62*0Sstevel@tonic-gate { 63*0Sstevel@tonic-gate uint64_t unscale = 0; 64*0Sstevel@tonic-gate hrtime_t rescale; 65*0Sstevel@tonic-gate hrtime_t diff = ts; 66*0Sstevel@tonic-gate 67*0Sstevel@tonic-gate while (diff > nsec_per_sys_tick) { 68*0Sstevel@tonic-gate unscale += hrtime2tick(diff); 69*0Sstevel@tonic-gate rescale = unscale; 70*0Sstevel@tonic-gate scalehrtime(&rescale); 71*0Sstevel@tonic-gate diff = ts - rescale; 72*0Sstevel@tonic-gate } 73*0Sstevel@tonic-gate 74*0Sstevel@tonic-gate return (unscale); 75*0Sstevel@tonic-gate } 76*0Sstevel@tonic-gate 77*0Sstevel@tonic-gate static int 78*0Sstevel@tonic-gate cbe_level1() 79*0Sstevel@tonic-gate { 80*0Sstevel@tonic-gate cyclic_softint(CPU, CY_LOW_LEVEL); 81*0Sstevel@tonic-gate return (1); 82*0Sstevel@tonic-gate } 83*0Sstevel@tonic-gate 84*0Sstevel@tonic-gate static int 85*0Sstevel@tonic-gate cbe_level10() 86*0Sstevel@tonic-gate { 87*0Sstevel@tonic-gate cyclic_softint(CPU, CY_LOCK_LEVEL); 88*0Sstevel@tonic-gate return (1); 89*0Sstevel@tonic-gate } 90*0Sstevel@tonic-gate 91*0Sstevel@tonic-gate /*ARGSUSED*/ 92*0Sstevel@tonic-gate static void 93*0Sstevel@tonic-gate cbe_enable(cyb_arg_t arg) 94*0Sstevel@tonic-gate { 95*0Sstevel@tonic-gate int pstate_save = disable_vec_intr(); 96*0Sstevel@tonic-gate 97*0Sstevel@tonic-gate intr_enqueue_req(PIL_14, cbe_level14_inum); 98*0Sstevel@tonic-gate enable_vec_intr(pstate_save); 99*0Sstevel@tonic-gate } 100*0Sstevel@tonic-gate 101*0Sstevel@tonic-gate /*ARGSUSED*/ 102*0Sstevel@tonic-gate static void 103*0Sstevel@tonic-gate cbe_disable(cyb_arg_t arg) 104*0Sstevel@tonic-gate { 105*0Sstevel@tonic-gate int pstate_save = disable_vec_intr(); 106*0Sstevel@tonic-gate 107*0Sstevel@tonic-gate tickcmpr_disable(); 108*0Sstevel@tonic-gate intr_dequeue_req(PIL_14, cbe_level14_inum); 109*0Sstevel@tonic-gate enable_vec_intr(pstate_save); 110*0Sstevel@tonic-gate } 111*0Sstevel@tonic-gate 112*0Sstevel@tonic-gate /*ARGSUSED*/ 113*0Sstevel@tonic-gate static void 114*0Sstevel@tonic-gate cbe_reprogram(cyb_arg_t arg, hrtime_t time) 115*0Sstevel@tonic-gate { 116*0Sstevel@tonic-gate if (time >= cbe_hrtime_max) 117*0Sstevel@tonic-gate time = cbe_hrtime_max; 118*0Sstevel@tonic-gate 119*0Sstevel@tonic-gate tickcmpr_set(unscalehrtime(time)); 120*0Sstevel@tonic-gate } 121*0Sstevel@tonic-gate 122*0Sstevel@tonic-gate static void 123*0Sstevel@tonic-gate cbe_softint(cyb_arg_t arg, cyc_level_t level) 124*0Sstevel@tonic-gate { 125*0Sstevel@tonic-gate cbe_data_t *data = (cbe_data_t *)arg; 126*0Sstevel@tonic-gate 127*0Sstevel@tonic-gate switch (level) { 128*0Sstevel@tonic-gate case CY_LOW_LEVEL: 129*0Sstevel@tonic-gate setsoftint(data->cbe_level1_inum); 130*0Sstevel@tonic-gate break; 131*0Sstevel@tonic-gate case CY_LOCK_LEVEL: 132*0Sstevel@tonic-gate setsoftint(data->cbe_level10_inum); 133*0Sstevel@tonic-gate break; 134*0Sstevel@tonic-gate default: 135*0Sstevel@tonic-gate panic("cbe_softint: unexpected soft level %d", level); 136*0Sstevel@tonic-gate } 137*0Sstevel@tonic-gate } 138*0Sstevel@tonic-gate 139*0Sstevel@tonic-gate /*ARGSUSED*/ 140*0Sstevel@tonic-gate static cyc_cookie_t 141*0Sstevel@tonic-gate cbe_set_level(cyb_arg_t arg, cyc_level_t level) 142*0Sstevel@tonic-gate { 143*0Sstevel@tonic-gate int ipl; 144*0Sstevel@tonic-gate 145*0Sstevel@tonic-gate switch (level) { 146*0Sstevel@tonic-gate case CY_LOW_LEVEL: 147*0Sstevel@tonic-gate ipl = CBE_LOW_PIL; 148*0Sstevel@tonic-gate break; 149*0Sstevel@tonic-gate case CY_LOCK_LEVEL: 150*0Sstevel@tonic-gate ipl = CBE_LOCK_PIL; 151*0Sstevel@tonic-gate break; 152*0Sstevel@tonic-gate case CY_HIGH_LEVEL: 153*0Sstevel@tonic-gate ipl = CBE_HIGH_PIL; 154*0Sstevel@tonic-gate break; 155*0Sstevel@tonic-gate default: 156*0Sstevel@tonic-gate panic("cbe_set_level: unexpected level %d", level); 157*0Sstevel@tonic-gate } 158*0Sstevel@tonic-gate 159*0Sstevel@tonic-gate return (splr(ipl)); 160*0Sstevel@tonic-gate } 161*0Sstevel@tonic-gate 162*0Sstevel@tonic-gate /*ARGSUSED*/ 163*0Sstevel@tonic-gate static void 164*0Sstevel@tonic-gate cbe_restore_level(cyb_arg_t arg, cyc_cookie_t cookie) 165*0Sstevel@tonic-gate { 166*0Sstevel@tonic-gate splx(cookie); 167*0Sstevel@tonic-gate } 168*0Sstevel@tonic-gate 169*0Sstevel@tonic-gate static void 170*0Sstevel@tonic-gate cbe_xcall_handler(uint64_t arg1, uint64_t arg2) 171*0Sstevel@tonic-gate { 172*0Sstevel@tonic-gate cyc_func_t func = (cyc_func_t)arg1; 173*0Sstevel@tonic-gate void *arg = (void *)arg2; 174*0Sstevel@tonic-gate 175*0Sstevel@tonic-gate (*func)(arg); 176*0Sstevel@tonic-gate } 177*0Sstevel@tonic-gate 178*0Sstevel@tonic-gate /*ARGSUSED*/ 179*0Sstevel@tonic-gate static void 180*0Sstevel@tonic-gate cbe_xcall(cyb_arg_t arg, cpu_t *dest, cyc_func_t func, void *farg) 181*0Sstevel@tonic-gate { 182*0Sstevel@tonic-gate kpreempt_disable(); 183*0Sstevel@tonic-gate xc_one(dest->cpu_id, cbe_xcall_handler, (uint64_t)func, (uint64_t)farg); 184*0Sstevel@tonic-gate kpreempt_enable(); 185*0Sstevel@tonic-gate } 186*0Sstevel@tonic-gate 187*0Sstevel@tonic-gate /*ARGSUSED*/ 188*0Sstevel@tonic-gate static cyb_arg_t 189*0Sstevel@tonic-gate cbe_configure(cpu_t *cpu) 190*0Sstevel@tonic-gate { 191*0Sstevel@tonic-gate cbe_data_t *new_data = kmem_alloc(sizeof (cbe_data_t), KM_SLEEP); 192*0Sstevel@tonic-gate 193*0Sstevel@tonic-gate /* 194*0Sstevel@tonic-gate * The setsoftint() code will refuse to post a soft interrupt if 195*0Sstevel@tonic-gate * one is already pending for the specified inum. Given that we 196*0Sstevel@tonic-gate * may have disjoint soft interrupts on different CPUs posted 197*0Sstevel@tonic-gate * simultaneously, we allocate a new set of inums for each CPU. 198*0Sstevel@tonic-gate */ 199*0Sstevel@tonic-gate new_data->cbe_level10_inum = 200*0Sstevel@tonic-gate add_softintr(PIL_10, (softintrfunc)cbe_level10, 0); 201*0Sstevel@tonic-gate 202*0Sstevel@tonic-gate new_data->cbe_level1_inum = 203*0Sstevel@tonic-gate add_softintr(PIL_1, (softintrfunc)cbe_level1, 0); 204*0Sstevel@tonic-gate 205*0Sstevel@tonic-gate return (new_data); 206*0Sstevel@tonic-gate } 207*0Sstevel@tonic-gate 208*0Sstevel@tonic-gate static void 209*0Sstevel@tonic-gate cbe_unconfigure(cyb_arg_t arg) 210*0Sstevel@tonic-gate { 211*0Sstevel@tonic-gate cbe_data_t *data = (cbe_data_t *)arg; 212*0Sstevel@tonic-gate 213*0Sstevel@tonic-gate rem_softintr(data->cbe_level10_inum); 214*0Sstevel@tonic-gate rem_softintr(data->cbe_level1_inum); 215*0Sstevel@tonic-gate 216*0Sstevel@tonic-gate kmem_free(data, sizeof (cbe_data_t)); 217*0Sstevel@tonic-gate } 218*0Sstevel@tonic-gate 219*0Sstevel@tonic-gate /*ARGSUSED*/ 220*0Sstevel@tonic-gate static void 221*0Sstevel@tonic-gate cbe_suspend(cyb_arg_t arg) 222*0Sstevel@tonic-gate { 223*0Sstevel@tonic-gate cbe_suspend_time = gethrtime_unscaled(); 224*0Sstevel@tonic-gate cbe_suspend_delta = 0; 225*0Sstevel@tonic-gate } 226*0Sstevel@tonic-gate 227*0Sstevel@tonic-gate /*ARGSUSED*/ 228*0Sstevel@tonic-gate static void 229*0Sstevel@tonic-gate cbe_resume(cyb_arg_t arg) 230*0Sstevel@tonic-gate { 231*0Sstevel@tonic-gate hrtime_t now; 232*0Sstevel@tonic-gate 233*0Sstevel@tonic-gate /* 234*0Sstevel@tonic-gate * If we're actually on a CPU which has apparently had %tick zeroed, 235*0Sstevel@tonic-gate * we want to add cbe_suspend_delta to %tick. 236*0Sstevel@tonic-gate */ 237*0Sstevel@tonic-gate if ((now = gethrtime_unscaled()) < cbe_suspend_time) { 238*0Sstevel@tonic-gate 239*0Sstevel@tonic-gate if (cbe_suspend_delta == 0) { 240*0Sstevel@tonic-gate /* 241*0Sstevel@tonic-gate * We're the first CPU to be resumed. We want %tick 242*0Sstevel@tonic-gate * to be close to %tick when we suspended the system, 243*0Sstevel@tonic-gate * so we'll figure out the delta which needs to be 244*0Sstevel@tonic-gate * written to the register. All subsequent resumed 245*0Sstevel@tonic-gate * CPUs will write the same delta. 246*0Sstevel@tonic-gate */ 247*0Sstevel@tonic-gate cbe_suspend_delta = cbe_suspend_time - now; 248*0Sstevel@tonic-gate } 249*0Sstevel@tonic-gate 250*0Sstevel@tonic-gate tick_write_delta(cbe_suspend_delta); 251*0Sstevel@tonic-gate } 252*0Sstevel@tonic-gate } 253*0Sstevel@tonic-gate 254*0Sstevel@tonic-gate void 255*0Sstevel@tonic-gate cbe_hres_tick(void) 256*0Sstevel@tonic-gate { 257*0Sstevel@tonic-gate dtrace_hres_tick(); 258*0Sstevel@tonic-gate hres_tick(); 259*0Sstevel@tonic-gate } 260*0Sstevel@tonic-gate 261*0Sstevel@tonic-gate void 262*0Sstevel@tonic-gate cbe_init(void) 263*0Sstevel@tonic-gate { 264*0Sstevel@tonic-gate cyc_handler_t hdlr; 265*0Sstevel@tonic-gate cyc_time_t when; 266*0Sstevel@tonic-gate hrtime_t resolution = NANOSEC / sys_tick_freq; 267*0Sstevel@tonic-gate 268*0Sstevel@tonic-gate cyc_backend_t cbe = { 269*0Sstevel@tonic-gate cbe_configure, /* cyb_configure */ 270*0Sstevel@tonic-gate cbe_unconfigure, /* cyb_unconfigure */ 271*0Sstevel@tonic-gate cbe_enable, /* cyb_enable */ 272*0Sstevel@tonic-gate cbe_disable, /* cyb_disable */ 273*0Sstevel@tonic-gate cbe_reprogram, /* cyb_reprogram */ 274*0Sstevel@tonic-gate cbe_softint, /* cyb_softint */ 275*0Sstevel@tonic-gate cbe_set_level, /* cyb_set_level */ 276*0Sstevel@tonic-gate cbe_restore_level, /* cyb_restore_level */ 277*0Sstevel@tonic-gate cbe_xcall, /* cyb_xcall */ 278*0Sstevel@tonic-gate cbe_suspend, /* cyb_suspend */ 279*0Sstevel@tonic-gate cbe_resume /* cyb_resume */ 280*0Sstevel@tonic-gate }; 281*0Sstevel@tonic-gate 282*0Sstevel@tonic-gate cbe_level14_inum = 283*0Sstevel@tonic-gate add_softintr(CBE_HIGH_PIL, (softintrfunc)cbe_level14, 0); 284*0Sstevel@tonic-gate cbe_hrtime_max = gethrtime_max(); 285*0Sstevel@tonic-gate 286*0Sstevel@tonic-gate /* 287*0Sstevel@tonic-gate * If sys_tick_freq > NANOSEC (i.e. we're on a CPU with a clock rate 288*0Sstevel@tonic-gate * which exceeds 1 GHz), we'll specify the minimum resolution, 289*0Sstevel@tonic-gate * 1 nanosecond. 290*0Sstevel@tonic-gate */ 291*0Sstevel@tonic-gate if (resolution == 0) 292*0Sstevel@tonic-gate resolution = 1; 293*0Sstevel@tonic-gate 294*0Sstevel@tonic-gate mutex_enter(&cpu_lock); 295*0Sstevel@tonic-gate cyclic_init(&cbe, resolution); 296*0Sstevel@tonic-gate 297*0Sstevel@tonic-gate /* 298*0Sstevel@tonic-gate * Initialize hrtime_base and hres_last_tick to reasonable starting 299*0Sstevel@tonic-gate * values. 300*0Sstevel@tonic-gate */ 301*0Sstevel@tonic-gate hrtime_base = gethrtime(); 302*0Sstevel@tonic-gate hres_last_tick = gethrtime_unscaled(); 303*0Sstevel@tonic-gate 304*0Sstevel@tonic-gate hdlr.cyh_level = CY_HIGH_LEVEL; 305*0Sstevel@tonic-gate hdlr.cyh_func = (cyc_func_t)cbe_hres_tick; 306*0Sstevel@tonic-gate hdlr.cyh_arg = NULL; 307*0Sstevel@tonic-gate 308*0Sstevel@tonic-gate when.cyt_when = 0; 309*0Sstevel@tonic-gate when.cyt_interval = nsec_per_tick; 310*0Sstevel@tonic-gate 311*0Sstevel@tonic-gate cbe_hres_cyclic = cyclic_add(&hdlr, &when); 312*0Sstevel@tonic-gate 313*0Sstevel@tonic-gate mutex_exit(&cpu_lock); 314*0Sstevel@tonic-gate 315*0Sstevel@tonic-gate clkstart(); 316*0Sstevel@tonic-gate } 317