1*0Sstevel@tonic-gate /* 2*0Sstevel@tonic-gate * CDDL HEADER START 3*0Sstevel@tonic-gate * 4*0Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*0Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*0Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*0Sstevel@tonic-gate * with the License. 8*0Sstevel@tonic-gate * 9*0Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*0Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*0Sstevel@tonic-gate * See the License for the specific language governing permissions 12*0Sstevel@tonic-gate * and limitations under the License. 13*0Sstevel@tonic-gate * 14*0Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*0Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*0Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*0Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*0Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*0Sstevel@tonic-gate * 20*0Sstevel@tonic-gate * CDDL HEADER END 21*0Sstevel@tonic-gate */ 22*0Sstevel@tonic-gate /* 23*0Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24*0Sstevel@tonic-gate * Use is subject to license terms. 25*0Sstevel@tonic-gate */ 26*0Sstevel@tonic-gate 27*0Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 28*0Sstevel@tonic-gate 29*0Sstevel@tonic-gate #include <sys/types.h> 30*0Sstevel@tonic-gate #include <sys/param.h> 31*0Sstevel@tonic-gate #include <sys/systm.h> 32*0Sstevel@tonic-gate #include <sys/user.h> 33*0Sstevel@tonic-gate #include <sys/proc.h> 34*0Sstevel@tonic-gate #include <sys/cpuvar.h> 35*0Sstevel@tonic-gate #include <sys/thread.h> 36*0Sstevel@tonic-gate #include <sys/debug.h> 37*0Sstevel@tonic-gate #include <sys/msacct.h> 38*0Sstevel@tonic-gate #include <sys/time.h> 39*0Sstevel@tonic-gate 40*0Sstevel@tonic-gate /* 41*0Sstevel@tonic-gate * Mega-theory block comment: 42*0Sstevel@tonic-gate * 43*0Sstevel@tonic-gate * Microstate accounting uses finite states and the transitions between these 44*0Sstevel@tonic-gate * states to measure timing and accounting information. The state information 45*0Sstevel@tonic-gate * is presently tracked for threads (via microstate accounting) and cpus (via 46*0Sstevel@tonic-gate * cpu microstate accounting). In each case, these accounting mechanisms use 47*0Sstevel@tonic-gate * states and transitions to measure time spent in each state instead of 48*0Sstevel@tonic-gate * clock-based sampling methodologies. 49*0Sstevel@tonic-gate * 50*0Sstevel@tonic-gate * For microstate accounting: 51*0Sstevel@tonic-gate * state transitions are accomplished by calling new_mstate() to switch between 52*0Sstevel@tonic-gate * states. Transitions from a sleeping state (LMS_SLEEP and LMS_STOPPED) occur 53*0Sstevel@tonic-gate * by calling restore_mstate() which restores a thread to its previously running 54*0Sstevel@tonic-gate * state. This code is primarialy executed by the dispatcher in disp() before 55*0Sstevel@tonic-gate * running a process that was put to sleep. If the thread was not in a sleeping 56*0Sstevel@tonic-gate * state, this call has little effect other than to update the count of time the 57*0Sstevel@tonic-gate * thread has spent waiting on run-queues in its lifetime. 58*0Sstevel@tonic-gate * 59*0Sstevel@tonic-gate * For cpu microstate accounting: 60*0Sstevel@tonic-gate * Cpu microstate accounting is similar to the microstate accounting for threads 61*0Sstevel@tonic-gate * but it tracks user, system, and idle time for cpus. Cpu microstate 62*0Sstevel@tonic-gate * accounting does not track interrupt times as there is a pre-existing 63*0Sstevel@tonic-gate * interrupt accounting mechanism for this purpose. Cpu microstate accounting 64*0Sstevel@tonic-gate * tracks time that user threads have spent active, idle, or in the system on a 65*0Sstevel@tonic-gate * given cpu. Cpu microstate accounting has fewer states which allows it to 66*0Sstevel@tonic-gate * have better defined transitions. The states transition in the following 67*0Sstevel@tonic-gate * order: 68*0Sstevel@tonic-gate * 69*0Sstevel@tonic-gate * CMS_USER <-> CMS_SYSTEM <-> CMS_IDLE 70*0Sstevel@tonic-gate * 71*0Sstevel@tonic-gate * In order to get to the idle state, the cpu microstate must first go through 72*0Sstevel@tonic-gate * the system state, and vice-versa for the user state from idle. The switching 73*0Sstevel@tonic-gate * of the microstates from user to system is done as part of the regular thread 74*0Sstevel@tonic-gate * microstate accounting code, except for the idle state which is switched by 75*0Sstevel@tonic-gate * the dispatcher before it runs the idle loop. 76*0Sstevel@tonic-gate * 77*0Sstevel@tonic-gate * Cpu percentages: 78*0Sstevel@tonic-gate * Cpu percentages are now handled by and based upon microstate accounting 79*0Sstevel@tonic-gate * information (the same is true for load averages). The routines which handle 80*0Sstevel@tonic-gate * the growing/shrinking and exponentiation of cpu percentages have been moved 81*0Sstevel@tonic-gate * here as it now makes more sense for them to be generated from the microstate 82*0Sstevel@tonic-gate * code. Cpu percentages are generated similarly to the way they were before; 83*0Sstevel@tonic-gate * however, now they are based upon high-resolution timestamps and the 84*0Sstevel@tonic-gate * timestamps are modified at various state changes instead of during a clock() 85*0Sstevel@tonic-gate * interrupt. This allows us to generate more accurate cpu percentages which 86*0Sstevel@tonic-gate * are also in-sync with microstate data. 87*0Sstevel@tonic-gate */ 88*0Sstevel@tonic-gate 89*0Sstevel@tonic-gate /* 90*0Sstevel@tonic-gate * Initialize the microstate level and the 91*0Sstevel@tonic-gate * associated accounting information for an LWP. 92*0Sstevel@tonic-gate */ 93*0Sstevel@tonic-gate void 94*0Sstevel@tonic-gate init_mstate( 95*0Sstevel@tonic-gate kthread_t *t, 96*0Sstevel@tonic-gate int init_state) 97*0Sstevel@tonic-gate { 98*0Sstevel@tonic-gate struct mstate *ms; 99*0Sstevel@tonic-gate klwp_t *lwp; 100*0Sstevel@tonic-gate hrtime_t curtime; 101*0Sstevel@tonic-gate 102*0Sstevel@tonic-gate ASSERT(init_state != LMS_WAIT_CPU); 103*0Sstevel@tonic-gate ASSERT((unsigned)init_state < NMSTATES); 104*0Sstevel@tonic-gate 105*0Sstevel@tonic-gate if ((lwp = ttolwp(t)) != NULL) { 106*0Sstevel@tonic-gate ms = &lwp->lwp_mstate; 107*0Sstevel@tonic-gate curtime = gethrtime_unscaled(); 108*0Sstevel@tonic-gate ms->ms_prev = LMS_SYSTEM; 109*0Sstevel@tonic-gate ms->ms_start = curtime; 110*0Sstevel@tonic-gate ms->ms_term = 0; 111*0Sstevel@tonic-gate ms->ms_state_start = curtime; 112*0Sstevel@tonic-gate t->t_mstate = init_state; 113*0Sstevel@tonic-gate t->t_waitrq = 0; 114*0Sstevel@tonic-gate t->t_hrtime = curtime; 115*0Sstevel@tonic-gate if ((t->t_proc_flag & TP_MSACCT) == 0) 116*0Sstevel@tonic-gate t->t_proc_flag |= TP_MSACCT; 117*0Sstevel@tonic-gate bzero((caddr_t)&ms->ms_acct[0], sizeof (ms->ms_acct)); 118*0Sstevel@tonic-gate } 119*0Sstevel@tonic-gate } 120*0Sstevel@tonic-gate 121*0Sstevel@tonic-gate /* 122*0Sstevel@tonic-gate * Initialize the microstate level and associated accounting information 123*0Sstevel@tonic-gate * for the specified cpu 124*0Sstevel@tonic-gate */ 125*0Sstevel@tonic-gate 126*0Sstevel@tonic-gate void 127*0Sstevel@tonic-gate init_cpu_mstate( 128*0Sstevel@tonic-gate cpu_t *cpu, 129*0Sstevel@tonic-gate int init_state) 130*0Sstevel@tonic-gate { 131*0Sstevel@tonic-gate ASSERT(init_state != CMS_DISABLED); 132*0Sstevel@tonic-gate 133*0Sstevel@tonic-gate cpu->cpu_mstate = init_state; 134*0Sstevel@tonic-gate cpu->cpu_mstate_start = gethrtime_unscaled(); 135*0Sstevel@tonic-gate cpu->cpu_waitrq = 0; 136*0Sstevel@tonic-gate bzero((caddr_t)&cpu->cpu_acct[0], sizeof (cpu->cpu_acct)); 137*0Sstevel@tonic-gate } 138*0Sstevel@tonic-gate 139*0Sstevel@tonic-gate /* 140*0Sstevel@tonic-gate * sets cpu state to OFFLINE. We don't actually track this time, 141*0Sstevel@tonic-gate * but it serves as a useful placeholder state for when we're not 142*0Sstevel@tonic-gate * doing anything. 143*0Sstevel@tonic-gate */ 144*0Sstevel@tonic-gate 145*0Sstevel@tonic-gate void 146*0Sstevel@tonic-gate term_cpu_mstate(struct cpu *cpu) 147*0Sstevel@tonic-gate { 148*0Sstevel@tonic-gate ASSERT(cpu->cpu_mstate != CMS_DISABLED); 149*0Sstevel@tonic-gate cpu->cpu_mstate = CMS_DISABLED; 150*0Sstevel@tonic-gate cpu->cpu_mstate_start = 0; 151*0Sstevel@tonic-gate } 152*0Sstevel@tonic-gate 153*0Sstevel@tonic-gate void 154*0Sstevel@tonic-gate new_cpu_mstate(cpu_t *cpu, int cmstate) 155*0Sstevel@tonic-gate { 156*0Sstevel@tonic-gate hrtime_t curtime; 157*0Sstevel@tonic-gate hrtime_t newtime; 158*0Sstevel@tonic-gate hrtime_t oldtime; 159*0Sstevel@tonic-gate hrtime_t *mstimep; 160*0Sstevel@tonic-gate 161*0Sstevel@tonic-gate ASSERT(cpu->cpu_mstate != CMS_DISABLED); 162*0Sstevel@tonic-gate ASSERT(cmstate < NCMSTATES); 163*0Sstevel@tonic-gate ASSERT(cmstate != CMS_DISABLED); 164*0Sstevel@tonic-gate ASSERT(curthread->t_preempt > 0 || curthread == cpu->cpu_idle_thread); 165*0Sstevel@tonic-gate 166*0Sstevel@tonic-gate curtime = gethrtime_unscaled(); 167*0Sstevel@tonic-gate mstimep = &cpu->cpu_acct[cpu->cpu_mstate]; 168*0Sstevel@tonic-gate do { 169*0Sstevel@tonic-gate newtime = curtime - cpu->cpu_mstate_start; 170*0Sstevel@tonic-gate if (newtime < 0) { 171*0Sstevel@tonic-gate /* force CAS to fail */ 172*0Sstevel@tonic-gate curtime = gethrtime_unscaled(); 173*0Sstevel@tonic-gate oldtime = *mstimep - 1; 174*0Sstevel@tonic-gate continue; 175*0Sstevel@tonic-gate } 176*0Sstevel@tonic-gate oldtime = *mstimep; 177*0Sstevel@tonic-gate newtime += oldtime; 178*0Sstevel@tonic-gate cpu->cpu_mstate = cmstate; 179*0Sstevel@tonic-gate cpu->cpu_mstate_start = curtime; 180*0Sstevel@tonic-gate } while (cas64((uint64_t *)mstimep, oldtime, newtime) != oldtime); 181*0Sstevel@tonic-gate } 182*0Sstevel@tonic-gate 183*0Sstevel@tonic-gate /* 184*0Sstevel@tonic-gate * Return an aggregation of microstate times in scaled nanoseconds (high-res 185*0Sstevel@tonic-gate * time). This keeps in mind that p_acct is already scaled, and ms_acct is 186*0Sstevel@tonic-gate * not. 187*0Sstevel@tonic-gate */ 188*0Sstevel@tonic-gate hrtime_t 189*0Sstevel@tonic-gate mstate_aggr_state(proc_t *p, int a_state) 190*0Sstevel@tonic-gate { 191*0Sstevel@tonic-gate struct mstate *ms; 192*0Sstevel@tonic-gate kthread_t *t; 193*0Sstevel@tonic-gate klwp_t *lwp; 194*0Sstevel@tonic-gate hrtime_t aggr_time; 195*0Sstevel@tonic-gate hrtime_t scaledtime; 196*0Sstevel@tonic-gate 197*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&p->p_lock)); 198*0Sstevel@tonic-gate ASSERT((unsigned)a_state < NMSTATES); 199*0Sstevel@tonic-gate 200*0Sstevel@tonic-gate aggr_time = p->p_acct[a_state]; 201*0Sstevel@tonic-gate if (a_state == LMS_SYSTEM) 202*0Sstevel@tonic-gate aggr_time += p->p_acct[LMS_TRAP]; 203*0Sstevel@tonic-gate 204*0Sstevel@tonic-gate t = p->p_tlist; 205*0Sstevel@tonic-gate if (t == NULL) 206*0Sstevel@tonic-gate return (aggr_time); 207*0Sstevel@tonic-gate 208*0Sstevel@tonic-gate do { 209*0Sstevel@tonic-gate if (t->t_proc_flag & TP_LWPEXIT) 210*0Sstevel@tonic-gate continue; 211*0Sstevel@tonic-gate 212*0Sstevel@tonic-gate lwp = ttolwp(t); 213*0Sstevel@tonic-gate ms = &lwp->lwp_mstate; 214*0Sstevel@tonic-gate scaledtime = ms->ms_acct[a_state]; 215*0Sstevel@tonic-gate scalehrtime(&scaledtime); 216*0Sstevel@tonic-gate aggr_time += scaledtime; 217*0Sstevel@tonic-gate if (a_state == LMS_SYSTEM) { 218*0Sstevel@tonic-gate scaledtime = ms->ms_acct[LMS_TRAP]; 219*0Sstevel@tonic-gate scalehrtime(&scaledtime); 220*0Sstevel@tonic-gate aggr_time += scaledtime; 221*0Sstevel@tonic-gate } 222*0Sstevel@tonic-gate } while ((t = t->t_forw) != p->p_tlist); 223*0Sstevel@tonic-gate 224*0Sstevel@tonic-gate return (aggr_time); 225*0Sstevel@tonic-gate } 226*0Sstevel@tonic-gate 227*0Sstevel@tonic-gate void 228*0Sstevel@tonic-gate syscall_mstate(int fromms, int toms) 229*0Sstevel@tonic-gate { 230*0Sstevel@tonic-gate kthread_t *t = curthread; 231*0Sstevel@tonic-gate struct mstate *ms; 232*0Sstevel@tonic-gate hrtime_t *mstimep; 233*0Sstevel@tonic-gate hrtime_t curtime; 234*0Sstevel@tonic-gate klwp_t *lwp; 235*0Sstevel@tonic-gate struct cpu *cpup; 236*0Sstevel@tonic-gate hrtime_t newtime; 237*0Sstevel@tonic-gate 238*0Sstevel@tonic-gate if ((lwp = ttolwp(t)) == NULL) 239*0Sstevel@tonic-gate return; 240*0Sstevel@tonic-gate 241*0Sstevel@tonic-gate ASSERT(fromms < NMSTATES); 242*0Sstevel@tonic-gate ASSERT(toms < NMSTATES); 243*0Sstevel@tonic-gate 244*0Sstevel@tonic-gate ms = &lwp->lwp_mstate; 245*0Sstevel@tonic-gate mstimep = &ms->ms_acct[fromms]; 246*0Sstevel@tonic-gate curtime = gethrtime_unscaled(); 247*0Sstevel@tonic-gate newtime = curtime - ms->ms_state_start; 248*0Sstevel@tonic-gate while (newtime < 0) { 249*0Sstevel@tonic-gate curtime = gethrtime_unscaled(); 250*0Sstevel@tonic-gate newtime = curtime - ms->ms_state_start; 251*0Sstevel@tonic-gate } 252*0Sstevel@tonic-gate *mstimep += newtime; 253*0Sstevel@tonic-gate t->t_mstate = toms; 254*0Sstevel@tonic-gate ms->ms_state_start = curtime; 255*0Sstevel@tonic-gate ms->ms_prev = fromms; 256*0Sstevel@tonic-gate /* 257*0Sstevel@tonic-gate * Here, you could call new_cpu_mstate() to switch the cpu 258*0Sstevel@tonic-gate * microstate. However, in the interest of making things 259*0Sstevel@tonic-gate * as expeditious as possible, the relevant work has been inlined. 260*0Sstevel@tonic-gate */ 261*0Sstevel@tonic-gate kpreempt_disable(); /* MUST disable kpreempt before touching t->cpu */ 262*0Sstevel@tonic-gate cpup = t->t_cpu; 263*0Sstevel@tonic-gate ASSERT(cpup->cpu_mstate != CMS_DISABLED); 264*0Sstevel@tonic-gate if ((toms != LMS_USER) && (cpup->cpu_mstate != CMS_SYSTEM)) { 265*0Sstevel@tonic-gate mstimep = &cpup->cpu_acct[CMS_USER]; 266*0Sstevel@tonic-gate newtime = curtime - cpup->cpu_mstate_start; 267*0Sstevel@tonic-gate while (newtime < 0) { 268*0Sstevel@tonic-gate curtime = gethrtime_unscaled(); 269*0Sstevel@tonic-gate newtime = curtime - cpup->cpu_mstate_start; 270*0Sstevel@tonic-gate } 271*0Sstevel@tonic-gate *mstimep += newtime; 272*0Sstevel@tonic-gate cpup->cpu_mstate = CMS_SYSTEM; 273*0Sstevel@tonic-gate cpup->cpu_mstate_start = curtime; 274*0Sstevel@tonic-gate } else if ((toms == LMS_USER) && (cpup->cpu_mstate != CMS_USER)) { 275*0Sstevel@tonic-gate mstimep = &cpup->cpu_acct[CMS_SYSTEM]; 276*0Sstevel@tonic-gate newtime = curtime - cpup->cpu_mstate_start; 277*0Sstevel@tonic-gate while (newtime < 0) { 278*0Sstevel@tonic-gate curtime = gethrtime_unscaled(); 279*0Sstevel@tonic-gate newtime = curtime - cpup->cpu_mstate_start; 280*0Sstevel@tonic-gate } 281*0Sstevel@tonic-gate *mstimep += newtime; 282*0Sstevel@tonic-gate cpup->cpu_mstate = CMS_USER; 283*0Sstevel@tonic-gate cpup->cpu_mstate_start = curtime; 284*0Sstevel@tonic-gate } 285*0Sstevel@tonic-gate kpreempt_enable(); 286*0Sstevel@tonic-gate } 287*0Sstevel@tonic-gate 288*0Sstevel@tonic-gate /* 289*0Sstevel@tonic-gate * The following is for computing the percentage of cpu time used recently 290*0Sstevel@tonic-gate * by an lwp. The function cpu_decay() is also called from /proc code. 291*0Sstevel@tonic-gate * 292*0Sstevel@tonic-gate * exp_x(x): 293*0Sstevel@tonic-gate * Given x as a 64-bit non-negative scaled integer of arbitrary magnitude, 294*0Sstevel@tonic-gate * Return exp(-x) as a 64-bit scaled integer in the range [0 .. 1]. 295*0Sstevel@tonic-gate * 296*0Sstevel@tonic-gate * Scaling for 64-bit scaled integer: 297*0Sstevel@tonic-gate * The binary point is to the right of the high-order bit 298*0Sstevel@tonic-gate * of the low-order 32-bit word. 299*0Sstevel@tonic-gate */ 300*0Sstevel@tonic-gate 301*0Sstevel@tonic-gate #define LSHIFT 31 302*0Sstevel@tonic-gate #define LSI_ONE ((uint32_t)1 << LSHIFT) /* 32-bit scaled integer 1 */ 303*0Sstevel@tonic-gate 304*0Sstevel@tonic-gate #ifdef DEBUG 305*0Sstevel@tonic-gate uint_t expx_cnt = 0; /* number of calls to exp_x() */ 306*0Sstevel@tonic-gate uint_t expx_mul = 0; /* number of long multiplies in exp_x() */ 307*0Sstevel@tonic-gate #endif 308*0Sstevel@tonic-gate 309*0Sstevel@tonic-gate static uint64_t 310*0Sstevel@tonic-gate exp_x(uint64_t x) 311*0Sstevel@tonic-gate { 312*0Sstevel@tonic-gate int i; 313*0Sstevel@tonic-gate uint64_t ull; 314*0Sstevel@tonic-gate uint32_t ui; 315*0Sstevel@tonic-gate 316*0Sstevel@tonic-gate #ifdef DEBUG 317*0Sstevel@tonic-gate expx_cnt++; 318*0Sstevel@tonic-gate #endif 319*0Sstevel@tonic-gate /* 320*0Sstevel@tonic-gate * By the formula: 321*0Sstevel@tonic-gate * exp(-x) = exp(-x/2) * exp(-x/2) 322*0Sstevel@tonic-gate * we keep halving x until it becomes small enough for 323*0Sstevel@tonic-gate * the following approximation to be accurate enough: 324*0Sstevel@tonic-gate * exp(-x) = 1 - x 325*0Sstevel@tonic-gate * We reduce x until it is less than 1/4 (the 2 in LSHIFT-2 below). 326*0Sstevel@tonic-gate * Our final error will be smaller than 4% . 327*0Sstevel@tonic-gate */ 328*0Sstevel@tonic-gate 329*0Sstevel@tonic-gate /* 330*0Sstevel@tonic-gate * Use a uint64_t for the initial shift calculation. 331*0Sstevel@tonic-gate */ 332*0Sstevel@tonic-gate ull = x >> (LSHIFT-2); 333*0Sstevel@tonic-gate 334*0Sstevel@tonic-gate /* 335*0Sstevel@tonic-gate * Short circuit: 336*0Sstevel@tonic-gate * A number this large produces effectively 0 (actually .005). 337*0Sstevel@tonic-gate * This way, we will never do more than 5 multiplies. 338*0Sstevel@tonic-gate */ 339*0Sstevel@tonic-gate if (ull >= (1 << 5)) 340*0Sstevel@tonic-gate return (0); 341*0Sstevel@tonic-gate 342*0Sstevel@tonic-gate ui = ull; /* OK. Now we can use a uint_t. */ 343*0Sstevel@tonic-gate for (i = 0; ui != 0; i++) 344*0Sstevel@tonic-gate ui >>= 1; 345*0Sstevel@tonic-gate 346*0Sstevel@tonic-gate if (i != 0) { 347*0Sstevel@tonic-gate #ifdef DEBUG 348*0Sstevel@tonic-gate expx_mul += i; /* seldom happens */ 349*0Sstevel@tonic-gate #endif 350*0Sstevel@tonic-gate x >>= i; 351*0Sstevel@tonic-gate } 352*0Sstevel@tonic-gate 353*0Sstevel@tonic-gate /* 354*0Sstevel@tonic-gate * Now we compute 1 - x and square it the number of times 355*0Sstevel@tonic-gate * that we halved x above to produce the final result: 356*0Sstevel@tonic-gate */ 357*0Sstevel@tonic-gate x = LSI_ONE - x; 358*0Sstevel@tonic-gate while (i--) 359*0Sstevel@tonic-gate x = (x * x) >> LSHIFT; 360*0Sstevel@tonic-gate 361*0Sstevel@tonic-gate return (x); 362*0Sstevel@tonic-gate } 363*0Sstevel@tonic-gate 364*0Sstevel@tonic-gate /* 365*0Sstevel@tonic-gate * Given the old percent cpu and a time delta in nanoseconds, 366*0Sstevel@tonic-gate * return the new decayed percent cpu: pct * exp(-tau), 367*0Sstevel@tonic-gate * where 'tau' is the time delta multiplied by a decay factor. 368*0Sstevel@tonic-gate * We have chosen the decay factor (cpu_decay_factor in param.c) 369*0Sstevel@tonic-gate * to make the decay over five seconds be approximately 20%. 370*0Sstevel@tonic-gate * 371*0Sstevel@tonic-gate * 'pct' is a 32-bit scaled integer <= 1 372*0Sstevel@tonic-gate * The binary point is to the right of the high-order bit 373*0Sstevel@tonic-gate * of the 32-bit word. 374*0Sstevel@tonic-gate */ 375*0Sstevel@tonic-gate static uint32_t 376*0Sstevel@tonic-gate cpu_decay(uint32_t pct, hrtime_t nsec) 377*0Sstevel@tonic-gate { 378*0Sstevel@tonic-gate uint64_t delta = (uint64_t)nsec; 379*0Sstevel@tonic-gate 380*0Sstevel@tonic-gate delta /= cpu_decay_factor; 381*0Sstevel@tonic-gate return ((pct * exp_x(delta)) >> LSHIFT); 382*0Sstevel@tonic-gate } 383*0Sstevel@tonic-gate 384*0Sstevel@tonic-gate /* 385*0Sstevel@tonic-gate * Given the old percent cpu and a time delta in nanoseconds, 386*0Sstevel@tonic-gate * return the new grown percent cpu: 1 - ( 1 - pct ) * exp(-tau) 387*0Sstevel@tonic-gate */ 388*0Sstevel@tonic-gate static uint32_t 389*0Sstevel@tonic-gate cpu_grow(uint32_t pct, hrtime_t nsec) 390*0Sstevel@tonic-gate { 391*0Sstevel@tonic-gate return (LSI_ONE - cpu_decay(LSI_ONE - pct, nsec)); 392*0Sstevel@tonic-gate } 393*0Sstevel@tonic-gate 394*0Sstevel@tonic-gate 395*0Sstevel@tonic-gate /* 396*0Sstevel@tonic-gate * Defined to determine whether a lwp is still on a processor. 397*0Sstevel@tonic-gate */ 398*0Sstevel@tonic-gate 399*0Sstevel@tonic-gate #define T_ONPROC(kt) \ 400*0Sstevel@tonic-gate ((kt)->t_mstate < LMS_SLEEP) 401*0Sstevel@tonic-gate #define T_OFFPROC(kt) \ 402*0Sstevel@tonic-gate ((kt)->t_mstate >= LMS_SLEEP) 403*0Sstevel@tonic-gate 404*0Sstevel@tonic-gate uint_t 405*0Sstevel@tonic-gate cpu_update_pct(kthread_t *t, hrtime_t newtime) 406*0Sstevel@tonic-gate { 407*0Sstevel@tonic-gate hrtime_t delta; 408*0Sstevel@tonic-gate hrtime_t hrlb; 409*0Sstevel@tonic-gate uint_t pctcpu; 410*0Sstevel@tonic-gate uint_t npctcpu; 411*0Sstevel@tonic-gate 412*0Sstevel@tonic-gate /* 413*0Sstevel@tonic-gate * This routine can get called at PIL > 0, this *has* to be 414*0Sstevel@tonic-gate * done atomically. Holding locks here causes bad things to happen. 415*0Sstevel@tonic-gate * (read: deadlock). 416*0Sstevel@tonic-gate */ 417*0Sstevel@tonic-gate 418*0Sstevel@tonic-gate do { 419*0Sstevel@tonic-gate if (T_ONPROC(t) && t->t_waitrq == 0) { 420*0Sstevel@tonic-gate hrlb = t->t_hrtime; 421*0Sstevel@tonic-gate delta = newtime - hrlb; 422*0Sstevel@tonic-gate if (delta < 0) { 423*0Sstevel@tonic-gate newtime = gethrtime_unscaled(); 424*0Sstevel@tonic-gate delta = newtime - hrlb; 425*0Sstevel@tonic-gate } 426*0Sstevel@tonic-gate t->t_hrtime = newtime; 427*0Sstevel@tonic-gate scalehrtime(&delta); 428*0Sstevel@tonic-gate pctcpu = t->t_pctcpu; 429*0Sstevel@tonic-gate npctcpu = cpu_grow(pctcpu, delta); 430*0Sstevel@tonic-gate } else { 431*0Sstevel@tonic-gate hrlb = t->t_hrtime; 432*0Sstevel@tonic-gate delta = newtime - hrlb; 433*0Sstevel@tonic-gate if (delta < 0) { 434*0Sstevel@tonic-gate newtime = gethrtime_unscaled(); 435*0Sstevel@tonic-gate delta = newtime - hrlb; 436*0Sstevel@tonic-gate } 437*0Sstevel@tonic-gate t->t_hrtime = newtime; 438*0Sstevel@tonic-gate scalehrtime(&delta); 439*0Sstevel@tonic-gate pctcpu = t->t_pctcpu; 440*0Sstevel@tonic-gate npctcpu = cpu_decay(pctcpu, delta); 441*0Sstevel@tonic-gate } 442*0Sstevel@tonic-gate } while (cas32(&t->t_pctcpu, pctcpu, npctcpu) != pctcpu); 443*0Sstevel@tonic-gate 444*0Sstevel@tonic-gate return (npctcpu); 445*0Sstevel@tonic-gate } 446*0Sstevel@tonic-gate 447*0Sstevel@tonic-gate /* 448*0Sstevel@tonic-gate * Change the microstate level for the LWP and update the 449*0Sstevel@tonic-gate * associated accounting information. Return the previous 450*0Sstevel@tonic-gate * LWP state. 451*0Sstevel@tonic-gate */ 452*0Sstevel@tonic-gate int 453*0Sstevel@tonic-gate new_mstate(kthread_t *t, int new_state) 454*0Sstevel@tonic-gate { 455*0Sstevel@tonic-gate struct mstate *ms; 456*0Sstevel@tonic-gate unsigned state; 457*0Sstevel@tonic-gate hrtime_t *mstimep; 458*0Sstevel@tonic-gate hrtime_t curtime; 459*0Sstevel@tonic-gate hrtime_t newtime; 460*0Sstevel@tonic-gate hrtime_t oldtime; 461*0Sstevel@tonic-gate klwp_t *lwp; 462*0Sstevel@tonic-gate 463*0Sstevel@tonic-gate ASSERT(new_state != LMS_WAIT_CPU); 464*0Sstevel@tonic-gate ASSERT((unsigned)new_state < NMSTATES); 465*0Sstevel@tonic-gate ASSERT(t == curthread || THREAD_LOCK_HELD(t)); 466*0Sstevel@tonic-gate 467*0Sstevel@tonic-gate if ((lwp = ttolwp(t)) == NULL) 468*0Sstevel@tonic-gate return (LMS_SYSTEM); 469*0Sstevel@tonic-gate 470*0Sstevel@tonic-gate curtime = gethrtime_unscaled(); 471*0Sstevel@tonic-gate 472*0Sstevel@tonic-gate /* adjust cpu percentages before we go any further */ 473*0Sstevel@tonic-gate (void) cpu_update_pct(t, curtime); 474*0Sstevel@tonic-gate 475*0Sstevel@tonic-gate ms = &lwp->lwp_mstate; 476*0Sstevel@tonic-gate state = t->t_mstate; 477*0Sstevel@tonic-gate do { 478*0Sstevel@tonic-gate switch (state) { 479*0Sstevel@tonic-gate case LMS_TFAULT: 480*0Sstevel@tonic-gate case LMS_DFAULT: 481*0Sstevel@tonic-gate case LMS_KFAULT: 482*0Sstevel@tonic-gate case LMS_USER_LOCK: 483*0Sstevel@tonic-gate mstimep = &ms->ms_acct[LMS_SYSTEM]; 484*0Sstevel@tonic-gate break; 485*0Sstevel@tonic-gate default: 486*0Sstevel@tonic-gate mstimep = &ms->ms_acct[state]; 487*0Sstevel@tonic-gate break; 488*0Sstevel@tonic-gate } 489*0Sstevel@tonic-gate newtime = curtime - ms->ms_state_start; 490*0Sstevel@tonic-gate if (newtime < 0) { 491*0Sstevel@tonic-gate curtime = gethrtime_unscaled(); 492*0Sstevel@tonic-gate oldtime = *mstimep - 1; /* force CAS to fail */ 493*0Sstevel@tonic-gate continue; 494*0Sstevel@tonic-gate } 495*0Sstevel@tonic-gate oldtime = *mstimep; 496*0Sstevel@tonic-gate newtime += oldtime; 497*0Sstevel@tonic-gate t->t_mstate = new_state; 498*0Sstevel@tonic-gate ms->ms_state_start = curtime; 499*0Sstevel@tonic-gate } while (cas64((uint64_t *)mstimep, oldtime, newtime) != oldtime); 500*0Sstevel@tonic-gate /* 501*0Sstevel@tonic-gate * Remember the previous running microstate. 502*0Sstevel@tonic-gate */ 503*0Sstevel@tonic-gate if (state != LMS_SLEEP && state != LMS_STOPPED) 504*0Sstevel@tonic-gate ms->ms_prev = state; 505*0Sstevel@tonic-gate 506*0Sstevel@tonic-gate /* 507*0Sstevel@tonic-gate * Switch CPU microstate if appropriate 508*0Sstevel@tonic-gate */ 509*0Sstevel@tonic-gate kpreempt_disable(); /* MUST disable kpreempt before touching t->cpu */ 510*0Sstevel@tonic-gate if (new_state == LMS_USER && t->t_cpu->cpu_mstate != CMS_USER) { 511*0Sstevel@tonic-gate new_cpu_mstate(t->t_cpu, CMS_USER); 512*0Sstevel@tonic-gate } else if (new_state != LMS_USER && 513*0Sstevel@tonic-gate t->t_cpu->cpu_mstate != CMS_SYSTEM) { 514*0Sstevel@tonic-gate new_cpu_mstate(t->t_cpu, CMS_SYSTEM); 515*0Sstevel@tonic-gate } 516*0Sstevel@tonic-gate kpreempt_enable(); 517*0Sstevel@tonic-gate 518*0Sstevel@tonic-gate return (ms->ms_prev); 519*0Sstevel@tonic-gate } 520*0Sstevel@tonic-gate 521*0Sstevel@tonic-gate static long waitrqis0 = 0; 522*0Sstevel@tonic-gate 523*0Sstevel@tonic-gate /* 524*0Sstevel@tonic-gate * Restore the LWP microstate to the previous runnable state. 525*0Sstevel@tonic-gate * Called from disp() with the newly selected lwp. 526*0Sstevel@tonic-gate */ 527*0Sstevel@tonic-gate void 528*0Sstevel@tonic-gate restore_mstate(kthread_t *t) 529*0Sstevel@tonic-gate { 530*0Sstevel@tonic-gate struct mstate *ms; 531*0Sstevel@tonic-gate hrtime_t *mstimep; 532*0Sstevel@tonic-gate klwp_t *lwp; 533*0Sstevel@tonic-gate hrtime_t curtime; 534*0Sstevel@tonic-gate hrtime_t waitrq; 535*0Sstevel@tonic-gate hrtime_t newtime; 536*0Sstevel@tonic-gate hrtime_t oldtime; 537*0Sstevel@tonic-gate struct cpu *cpup; 538*0Sstevel@tonic-gate 539*0Sstevel@tonic-gate if ((lwp = ttolwp(t)) == NULL) 540*0Sstevel@tonic-gate return; 541*0Sstevel@tonic-gate 542*0Sstevel@tonic-gate curtime = gethrtime_unscaled(); 543*0Sstevel@tonic-gate (void) cpu_update_pct(t, curtime); 544*0Sstevel@tonic-gate ms = &lwp->lwp_mstate; 545*0Sstevel@tonic-gate ASSERT((unsigned)t->t_mstate < NMSTATES); 546*0Sstevel@tonic-gate do { 547*0Sstevel@tonic-gate switch (t->t_mstate) { 548*0Sstevel@tonic-gate case LMS_SLEEP: 549*0Sstevel@tonic-gate /* 550*0Sstevel@tonic-gate * Update the timer for the current sleep state. 551*0Sstevel@tonic-gate */ 552*0Sstevel@tonic-gate ASSERT((unsigned)ms->ms_prev < NMSTATES); 553*0Sstevel@tonic-gate switch (ms->ms_prev) { 554*0Sstevel@tonic-gate case LMS_TFAULT: 555*0Sstevel@tonic-gate case LMS_DFAULT: 556*0Sstevel@tonic-gate case LMS_KFAULT: 557*0Sstevel@tonic-gate case LMS_USER_LOCK: 558*0Sstevel@tonic-gate mstimep = &ms->ms_acct[ms->ms_prev]; 559*0Sstevel@tonic-gate break; 560*0Sstevel@tonic-gate default: 561*0Sstevel@tonic-gate mstimep = &ms->ms_acct[LMS_SLEEP]; 562*0Sstevel@tonic-gate break; 563*0Sstevel@tonic-gate } 564*0Sstevel@tonic-gate /* 565*0Sstevel@tonic-gate * Return to the previous run state. 566*0Sstevel@tonic-gate */ 567*0Sstevel@tonic-gate t->t_mstate = ms->ms_prev; 568*0Sstevel@tonic-gate break; 569*0Sstevel@tonic-gate case LMS_STOPPED: 570*0Sstevel@tonic-gate mstimep = &ms->ms_acct[LMS_STOPPED]; 571*0Sstevel@tonic-gate /* 572*0Sstevel@tonic-gate * Return to the previous run state. 573*0Sstevel@tonic-gate */ 574*0Sstevel@tonic-gate t->t_mstate = ms->ms_prev; 575*0Sstevel@tonic-gate break; 576*0Sstevel@tonic-gate case LMS_TFAULT: 577*0Sstevel@tonic-gate case LMS_DFAULT: 578*0Sstevel@tonic-gate case LMS_KFAULT: 579*0Sstevel@tonic-gate case LMS_USER_LOCK: 580*0Sstevel@tonic-gate mstimep = &ms->ms_acct[LMS_SYSTEM]; 581*0Sstevel@tonic-gate break; 582*0Sstevel@tonic-gate default: 583*0Sstevel@tonic-gate mstimep = &ms->ms_acct[t->t_mstate]; 584*0Sstevel@tonic-gate break; 585*0Sstevel@tonic-gate } 586*0Sstevel@tonic-gate waitrq = t->t_waitrq; /* hopefully atomic */ 587*0Sstevel@tonic-gate t->t_waitrq = 0; 588*0Sstevel@tonic-gate if (waitrq == 0) { /* should only happen during boot */ 589*0Sstevel@tonic-gate waitrq = curtime; 590*0Sstevel@tonic-gate waitrqis0++; 591*0Sstevel@tonic-gate } 592*0Sstevel@tonic-gate newtime = waitrq - ms->ms_state_start; 593*0Sstevel@tonic-gate if (newtime < 0) { 594*0Sstevel@tonic-gate curtime = gethrtime_unscaled(); 595*0Sstevel@tonic-gate oldtime = *mstimep - 1; /* force CAS to fail */ 596*0Sstevel@tonic-gate continue; 597*0Sstevel@tonic-gate } 598*0Sstevel@tonic-gate oldtime = *mstimep; 599*0Sstevel@tonic-gate newtime += oldtime; 600*0Sstevel@tonic-gate } while (cas64((uint64_t *)mstimep, oldtime, newtime) != oldtime); 601*0Sstevel@tonic-gate /* 602*0Sstevel@tonic-gate * Update the WAIT_CPU timer and per-cpu waitrq total. 603*0Sstevel@tonic-gate */ 604*0Sstevel@tonic-gate cpup = t->t_disp_queue->disp_cpu; 605*0Sstevel@tonic-gate if (cpup == NULL) 606*0Sstevel@tonic-gate cpup = t->t_cpu; 607*0Sstevel@tonic-gate ms->ms_acct[LMS_WAIT_CPU] += (curtime - waitrq); 608*0Sstevel@tonic-gate cpup->cpu_waitrq += (curtime - waitrq); 609*0Sstevel@tonic-gate ms->ms_state_start = curtime; 610*0Sstevel@tonic-gate } 611*0Sstevel@tonic-gate 612*0Sstevel@tonic-gate /* 613*0Sstevel@tonic-gate * Copy lwp microstate accounting and resource usage information 614*0Sstevel@tonic-gate * to the process. (lwp is terminating) 615*0Sstevel@tonic-gate */ 616*0Sstevel@tonic-gate void 617*0Sstevel@tonic-gate term_mstate(kthread_t *t) 618*0Sstevel@tonic-gate { 619*0Sstevel@tonic-gate struct mstate *ms; 620*0Sstevel@tonic-gate proc_t *p = ttoproc(t); 621*0Sstevel@tonic-gate klwp_t *lwp = ttolwp(t); 622*0Sstevel@tonic-gate int i; 623*0Sstevel@tonic-gate hrtime_t tmp; 624*0Sstevel@tonic-gate 625*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&p->p_lock)); 626*0Sstevel@tonic-gate 627*0Sstevel@tonic-gate ms = &lwp->lwp_mstate; 628*0Sstevel@tonic-gate (void) new_mstate(t, LMS_STOPPED); 629*0Sstevel@tonic-gate ms->ms_term = ms->ms_state_start; 630*0Sstevel@tonic-gate tmp = ms->ms_term - ms->ms_start; 631*0Sstevel@tonic-gate scalehrtime(&tmp); 632*0Sstevel@tonic-gate p->p_mlreal += tmp; 633*0Sstevel@tonic-gate for (i = 0; i < NMSTATES; i++) { 634*0Sstevel@tonic-gate tmp = ms->ms_acct[i]; 635*0Sstevel@tonic-gate scalehrtime(&tmp); 636*0Sstevel@tonic-gate p->p_acct[i] += tmp; 637*0Sstevel@tonic-gate } 638*0Sstevel@tonic-gate p->p_ru.minflt += lwp->lwp_ru.minflt; 639*0Sstevel@tonic-gate p->p_ru.majflt += lwp->lwp_ru.majflt; 640*0Sstevel@tonic-gate p->p_ru.nswap += lwp->lwp_ru.nswap; 641*0Sstevel@tonic-gate p->p_ru.inblock += lwp->lwp_ru.inblock; 642*0Sstevel@tonic-gate p->p_ru.oublock += lwp->lwp_ru.oublock; 643*0Sstevel@tonic-gate p->p_ru.msgsnd += lwp->lwp_ru.msgsnd; 644*0Sstevel@tonic-gate p->p_ru.msgrcv += lwp->lwp_ru.msgrcv; 645*0Sstevel@tonic-gate p->p_ru.nsignals += lwp->lwp_ru.nsignals; 646*0Sstevel@tonic-gate p->p_ru.nvcsw += lwp->lwp_ru.nvcsw; 647*0Sstevel@tonic-gate p->p_ru.nivcsw += lwp->lwp_ru.nivcsw; 648*0Sstevel@tonic-gate p->p_ru.sysc += lwp->lwp_ru.sysc; 649*0Sstevel@tonic-gate p->p_ru.ioch += lwp->lwp_ru.ioch; 650*0Sstevel@tonic-gate p->p_defunct++; 651*0Sstevel@tonic-gate } 652