1*0Sstevel@tonic-gate /* 2*0Sstevel@tonic-gate * CDDL HEADER START 3*0Sstevel@tonic-gate * 4*0Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*0Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*0Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*0Sstevel@tonic-gate * with the License. 8*0Sstevel@tonic-gate * 9*0Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*0Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*0Sstevel@tonic-gate * See the License for the specific language governing permissions 12*0Sstevel@tonic-gate * and limitations under the License. 13*0Sstevel@tonic-gate * 14*0Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*0Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*0Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*0Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*0Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*0Sstevel@tonic-gate * 20*0Sstevel@tonic-gate * CDDL HEADER END 21*0Sstevel@tonic-gate */ 22*0Sstevel@tonic-gate /* 23*0Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24*0Sstevel@tonic-gate * Use is subject to license terms. 25*0Sstevel@tonic-gate */ 26*0Sstevel@tonic-gate 27*0Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 28*0Sstevel@tonic-gate 29*0Sstevel@tonic-gate /* 30*0Sstevel@tonic-gate * Architecture-independent CPU control functions. 31*0Sstevel@tonic-gate */ 32*0Sstevel@tonic-gate 33*0Sstevel@tonic-gate #include <sys/types.h> 34*0Sstevel@tonic-gate #include <sys/param.h> 35*0Sstevel@tonic-gate #include <sys/var.h> 36*0Sstevel@tonic-gate #include <sys/thread.h> 37*0Sstevel@tonic-gate #include <sys/cpuvar.h> 38*0Sstevel@tonic-gate #include <sys/kstat.h> 39*0Sstevel@tonic-gate #include <sys/uadmin.h> 40*0Sstevel@tonic-gate #include <sys/systm.h> 41*0Sstevel@tonic-gate #include <sys/errno.h> 42*0Sstevel@tonic-gate #include <sys/cmn_err.h> 43*0Sstevel@tonic-gate #include <sys/procset.h> 44*0Sstevel@tonic-gate #include <sys/processor.h> 45*0Sstevel@tonic-gate #include <sys/debug.h> 46*0Sstevel@tonic-gate #include <sys/cpupart.h> 47*0Sstevel@tonic-gate #include <sys/lgrp.h> 48*0Sstevel@tonic-gate #include <sys/pset.h> 49*0Sstevel@tonic-gate #include <sys/chip.h> 50*0Sstevel@tonic-gate #include <sys/kmem.h> 51*0Sstevel@tonic-gate #include <sys/kmem_impl.h> /* to set per-cpu kmem_cache offset */ 52*0Sstevel@tonic-gate #include <sys/atomic.h> 53*0Sstevel@tonic-gate #include <sys/callb.h> 54*0Sstevel@tonic-gate #include <sys/vtrace.h> 55*0Sstevel@tonic-gate #include <sys/cyclic.h> 56*0Sstevel@tonic-gate #include <sys/bitmap.h> 57*0Sstevel@tonic-gate #include <sys/nvpair.h> 58*0Sstevel@tonic-gate #include <sys/pool_pset.h> 59*0Sstevel@tonic-gate #include <sys/msacct.h> 60*0Sstevel@tonic-gate #include <sys/time.h> 61*0Sstevel@tonic-gate #include <sys/archsystm.h> 62*0Sstevel@tonic-gate 63*0Sstevel@tonic-gate extern int mp_cpu_start(cpu_t *); 64*0Sstevel@tonic-gate extern int mp_cpu_stop(cpu_t *); 65*0Sstevel@tonic-gate extern int mp_cpu_poweron(cpu_t *); 66*0Sstevel@tonic-gate extern int mp_cpu_poweroff(cpu_t *); 67*0Sstevel@tonic-gate extern int mp_cpu_configure(int); 68*0Sstevel@tonic-gate extern int mp_cpu_unconfigure(int); 69*0Sstevel@tonic-gate extern void mp_cpu_faulted_enter(cpu_t *); 70*0Sstevel@tonic-gate extern void mp_cpu_faulted_exit(cpu_t *); 71*0Sstevel@tonic-gate 72*0Sstevel@tonic-gate extern int cmp_cpu_to_chip(processorid_t cpuid); 73*0Sstevel@tonic-gate #ifdef __sparcv9 74*0Sstevel@tonic-gate extern char *cpu_fru_fmri(cpu_t *cp); 75*0Sstevel@tonic-gate #endif 76*0Sstevel@tonic-gate 77*0Sstevel@tonic-gate static void cpu_add_active_internal(cpu_t *cp); 78*0Sstevel@tonic-gate static void cpu_remove_active(cpu_t *cp); 79*0Sstevel@tonic-gate static void cpu_info_kstat_create(cpu_t *cp); 80*0Sstevel@tonic-gate static void cpu_info_kstat_destroy(cpu_t *cp); 81*0Sstevel@tonic-gate static void cpu_stats_kstat_create(cpu_t *cp); 82*0Sstevel@tonic-gate static void cpu_stats_kstat_destroy(cpu_t *cp); 83*0Sstevel@tonic-gate 84*0Sstevel@tonic-gate static int cpu_sys_stats_ks_update(kstat_t *ksp, int rw); 85*0Sstevel@tonic-gate static int cpu_vm_stats_ks_update(kstat_t *ksp, int rw); 86*0Sstevel@tonic-gate static int cpu_stat_ks_update(kstat_t *ksp, int rw); 87*0Sstevel@tonic-gate static int cpu_state_change_hooks(int, cpu_setup_t, cpu_setup_t); 88*0Sstevel@tonic-gate 89*0Sstevel@tonic-gate /* 90*0Sstevel@tonic-gate * cpu_lock protects ncpus, ncpus_online, cpu_flag, cpu_list, cpu_active, 91*0Sstevel@tonic-gate * and dispatch queue reallocations. The lock ordering with respect to 92*0Sstevel@tonic-gate * related locks is: 93*0Sstevel@tonic-gate * 94*0Sstevel@tonic-gate * cpu_lock --> thread_free_lock ---> p_lock ---> thread_lock() 95*0Sstevel@tonic-gate * 96*0Sstevel@tonic-gate * Warning: Certain sections of code do not use the cpu_lock when 97*0Sstevel@tonic-gate * traversing the cpu_list (e.g. mutex_vector_enter(), clock()). Since 98*0Sstevel@tonic-gate * all cpus are paused during modifications to this list, a solution 99*0Sstevel@tonic-gate * to protect the list is too either disable kernel preemption while 100*0Sstevel@tonic-gate * walking the list, *or* recheck the cpu_next pointer at each 101*0Sstevel@tonic-gate * iteration in the loop. Note that in no cases can any cached 102*0Sstevel@tonic-gate * copies of the cpu pointers be kept as they may become invalid. 103*0Sstevel@tonic-gate */ 104*0Sstevel@tonic-gate kmutex_t cpu_lock; 105*0Sstevel@tonic-gate cpu_t *cpu_list; /* list of all CPUs */ 106*0Sstevel@tonic-gate cpu_t *cpu_active; /* list of active CPUs */ 107*0Sstevel@tonic-gate static cpuset_t cpu_available; /* set of available CPUs */ 108*0Sstevel@tonic-gate cpuset_t cpu_seqid_inuse; /* which cpu_seqids are in use */ 109*0Sstevel@tonic-gate 110*0Sstevel@tonic-gate /* 111*0Sstevel@tonic-gate * max_ncpus keeps the max cpus the system can have. Initially 112*0Sstevel@tonic-gate * it's NCPU, but since most archs scan the devtree for cpus 113*0Sstevel@tonic-gate * fairly early on during boot, the real max can be known before 114*0Sstevel@tonic-gate * ncpus is set (useful for early NCPU based allocations). 115*0Sstevel@tonic-gate */ 116*0Sstevel@tonic-gate int max_ncpus = NCPU; 117*0Sstevel@tonic-gate /* 118*0Sstevel@tonic-gate * platforms that set max_ncpus to maxiumum number of cpus that can be 119*0Sstevel@tonic-gate * dynamically added will set boot_max_ncpus to the number of cpus found 120*0Sstevel@tonic-gate * at device tree scan time during boot. 121*0Sstevel@tonic-gate */ 122*0Sstevel@tonic-gate int boot_max_ncpus = -1; 123*0Sstevel@tonic-gate /* 124*0Sstevel@tonic-gate * Maximum possible CPU id. This can never be >= NCPU since NCPU is 125*0Sstevel@tonic-gate * used to size arrays that are indexed by CPU id. 126*0Sstevel@tonic-gate */ 127*0Sstevel@tonic-gate processorid_t max_cpuid = NCPU - 1; 128*0Sstevel@tonic-gate 129*0Sstevel@tonic-gate int ncpus = 1; 130*0Sstevel@tonic-gate int ncpus_online = 1; 131*0Sstevel@tonic-gate 132*0Sstevel@tonic-gate /* 133*0Sstevel@tonic-gate * CPU that we're trying to offline. Protected by cpu_lock. 134*0Sstevel@tonic-gate */ 135*0Sstevel@tonic-gate cpu_t *cpu_inmotion; 136*0Sstevel@tonic-gate 137*0Sstevel@tonic-gate /* 138*0Sstevel@tonic-gate * Can be raised to suppress further weakbinding, which are instead 139*0Sstevel@tonic-gate * satisfied by disabling preemption. Must be raised/lowered under cpu_lock, 140*0Sstevel@tonic-gate * while individual thread weakbinding synchronisation is done under thread 141*0Sstevel@tonic-gate * lock. 142*0Sstevel@tonic-gate */ 143*0Sstevel@tonic-gate int weakbindingbarrier; 144*0Sstevel@tonic-gate 145*0Sstevel@tonic-gate /* 146*0Sstevel@tonic-gate * values for safe_list. Pause state that CPUs are in. 147*0Sstevel@tonic-gate */ 148*0Sstevel@tonic-gate #define PAUSE_IDLE 0 /* normal state */ 149*0Sstevel@tonic-gate #define PAUSE_READY 1 /* paused thread ready to spl */ 150*0Sstevel@tonic-gate #define PAUSE_WAIT 2 /* paused thread is spl-ed high */ 151*0Sstevel@tonic-gate #define PAUSE_DIE 3 /* tell pause thread to leave */ 152*0Sstevel@tonic-gate #define PAUSE_DEAD 4 /* pause thread has left */ 153*0Sstevel@tonic-gate 154*0Sstevel@tonic-gate /* 155*0Sstevel@tonic-gate * Variables used in pause_cpus(). 156*0Sstevel@tonic-gate */ 157*0Sstevel@tonic-gate static volatile char safe_list[NCPU]; 158*0Sstevel@tonic-gate 159*0Sstevel@tonic-gate static struct _cpu_pause_info { 160*0Sstevel@tonic-gate int cp_spl; /* spl saved in pause_cpus() */ 161*0Sstevel@tonic-gate volatile int cp_go; /* Go signal sent after all ready */ 162*0Sstevel@tonic-gate int cp_count; /* # of CPUs to pause */ 163*0Sstevel@tonic-gate ksema_t cp_sem; /* synch pause_cpus & cpu_pause */ 164*0Sstevel@tonic-gate kthread_id_t cp_paused; 165*0Sstevel@tonic-gate } cpu_pause_info; 166*0Sstevel@tonic-gate 167*0Sstevel@tonic-gate static kmutex_t pause_free_mutex; 168*0Sstevel@tonic-gate static kcondvar_t pause_free_cv; 169*0Sstevel@tonic-gate 170*0Sstevel@tonic-gate static struct cpu_sys_stats_ks_data { 171*0Sstevel@tonic-gate kstat_named_t cpu_ticks_idle; 172*0Sstevel@tonic-gate kstat_named_t cpu_ticks_user; 173*0Sstevel@tonic-gate kstat_named_t cpu_ticks_kernel; 174*0Sstevel@tonic-gate kstat_named_t cpu_ticks_wait; 175*0Sstevel@tonic-gate kstat_named_t cpu_nsec_idle; 176*0Sstevel@tonic-gate kstat_named_t cpu_nsec_user; 177*0Sstevel@tonic-gate kstat_named_t cpu_nsec_kernel; 178*0Sstevel@tonic-gate kstat_named_t wait_ticks_io; 179*0Sstevel@tonic-gate kstat_named_t bread; 180*0Sstevel@tonic-gate kstat_named_t bwrite; 181*0Sstevel@tonic-gate kstat_named_t lread; 182*0Sstevel@tonic-gate kstat_named_t lwrite; 183*0Sstevel@tonic-gate kstat_named_t phread; 184*0Sstevel@tonic-gate kstat_named_t phwrite; 185*0Sstevel@tonic-gate kstat_named_t pswitch; 186*0Sstevel@tonic-gate kstat_named_t trap; 187*0Sstevel@tonic-gate kstat_named_t intr; 188*0Sstevel@tonic-gate kstat_named_t syscall; 189*0Sstevel@tonic-gate kstat_named_t sysread; 190*0Sstevel@tonic-gate kstat_named_t syswrite; 191*0Sstevel@tonic-gate kstat_named_t sysfork; 192*0Sstevel@tonic-gate kstat_named_t sysvfork; 193*0Sstevel@tonic-gate kstat_named_t sysexec; 194*0Sstevel@tonic-gate kstat_named_t readch; 195*0Sstevel@tonic-gate kstat_named_t writech; 196*0Sstevel@tonic-gate kstat_named_t rcvint; 197*0Sstevel@tonic-gate kstat_named_t xmtint; 198*0Sstevel@tonic-gate kstat_named_t mdmint; 199*0Sstevel@tonic-gate kstat_named_t rawch; 200*0Sstevel@tonic-gate kstat_named_t canch; 201*0Sstevel@tonic-gate kstat_named_t outch; 202*0Sstevel@tonic-gate kstat_named_t msg; 203*0Sstevel@tonic-gate kstat_named_t sema; 204*0Sstevel@tonic-gate kstat_named_t namei; 205*0Sstevel@tonic-gate kstat_named_t ufsiget; 206*0Sstevel@tonic-gate kstat_named_t ufsdirblk; 207*0Sstevel@tonic-gate kstat_named_t ufsipage; 208*0Sstevel@tonic-gate kstat_named_t ufsinopage; 209*0Sstevel@tonic-gate kstat_named_t procovf; 210*0Sstevel@tonic-gate kstat_named_t intrthread; 211*0Sstevel@tonic-gate kstat_named_t intrblk; 212*0Sstevel@tonic-gate kstat_named_t intrunpin; 213*0Sstevel@tonic-gate kstat_named_t idlethread; 214*0Sstevel@tonic-gate kstat_named_t inv_swtch; 215*0Sstevel@tonic-gate kstat_named_t nthreads; 216*0Sstevel@tonic-gate kstat_named_t cpumigrate; 217*0Sstevel@tonic-gate kstat_named_t xcalls; 218*0Sstevel@tonic-gate kstat_named_t mutex_adenters; 219*0Sstevel@tonic-gate kstat_named_t rw_rdfails; 220*0Sstevel@tonic-gate kstat_named_t rw_wrfails; 221*0Sstevel@tonic-gate kstat_named_t modload; 222*0Sstevel@tonic-gate kstat_named_t modunload; 223*0Sstevel@tonic-gate kstat_named_t bawrite; 224*0Sstevel@tonic-gate kstat_named_t iowait; 225*0Sstevel@tonic-gate } cpu_sys_stats_ks_data_template = { 226*0Sstevel@tonic-gate { "cpu_ticks_idle", KSTAT_DATA_UINT64 }, 227*0Sstevel@tonic-gate { "cpu_ticks_user", KSTAT_DATA_UINT64 }, 228*0Sstevel@tonic-gate { "cpu_ticks_kernel", KSTAT_DATA_UINT64 }, 229*0Sstevel@tonic-gate { "cpu_ticks_wait", KSTAT_DATA_UINT64 }, 230*0Sstevel@tonic-gate { "cpu_nsec_idle", KSTAT_DATA_UINT64 }, 231*0Sstevel@tonic-gate { "cpu_nsec_user", KSTAT_DATA_UINT64 }, 232*0Sstevel@tonic-gate { "cpu_nsec_kernel", KSTAT_DATA_UINT64 }, 233*0Sstevel@tonic-gate { "wait_ticks_io", KSTAT_DATA_UINT64 }, 234*0Sstevel@tonic-gate { "bread", KSTAT_DATA_UINT64 }, 235*0Sstevel@tonic-gate { "bwrite", KSTAT_DATA_UINT64 }, 236*0Sstevel@tonic-gate { "lread", KSTAT_DATA_UINT64 }, 237*0Sstevel@tonic-gate { "lwrite", KSTAT_DATA_UINT64 }, 238*0Sstevel@tonic-gate { "phread", KSTAT_DATA_UINT64 }, 239*0Sstevel@tonic-gate { "phwrite", KSTAT_DATA_UINT64 }, 240*0Sstevel@tonic-gate { "pswitch", KSTAT_DATA_UINT64 }, 241*0Sstevel@tonic-gate { "trap", KSTAT_DATA_UINT64 }, 242*0Sstevel@tonic-gate { "intr", KSTAT_DATA_UINT64 }, 243*0Sstevel@tonic-gate { "syscall", KSTAT_DATA_UINT64 }, 244*0Sstevel@tonic-gate { "sysread", KSTAT_DATA_UINT64 }, 245*0Sstevel@tonic-gate { "syswrite", KSTAT_DATA_UINT64 }, 246*0Sstevel@tonic-gate { "sysfork", KSTAT_DATA_UINT64 }, 247*0Sstevel@tonic-gate { "sysvfork", KSTAT_DATA_UINT64 }, 248*0Sstevel@tonic-gate { "sysexec", KSTAT_DATA_UINT64 }, 249*0Sstevel@tonic-gate { "readch", KSTAT_DATA_UINT64 }, 250*0Sstevel@tonic-gate { "writech", KSTAT_DATA_UINT64 }, 251*0Sstevel@tonic-gate { "rcvint", KSTAT_DATA_UINT64 }, 252*0Sstevel@tonic-gate { "xmtint", KSTAT_DATA_UINT64 }, 253*0Sstevel@tonic-gate { "mdmint", KSTAT_DATA_UINT64 }, 254*0Sstevel@tonic-gate { "rawch", KSTAT_DATA_UINT64 }, 255*0Sstevel@tonic-gate { "canch", KSTAT_DATA_UINT64 }, 256*0Sstevel@tonic-gate { "outch", KSTAT_DATA_UINT64 }, 257*0Sstevel@tonic-gate { "msg", KSTAT_DATA_UINT64 }, 258*0Sstevel@tonic-gate { "sema", KSTAT_DATA_UINT64 }, 259*0Sstevel@tonic-gate { "namei", KSTAT_DATA_UINT64 }, 260*0Sstevel@tonic-gate { "ufsiget", KSTAT_DATA_UINT64 }, 261*0Sstevel@tonic-gate { "ufsdirblk", KSTAT_DATA_UINT64 }, 262*0Sstevel@tonic-gate { "ufsipage", KSTAT_DATA_UINT64 }, 263*0Sstevel@tonic-gate { "ufsinopage", KSTAT_DATA_UINT64 }, 264*0Sstevel@tonic-gate { "procovf", KSTAT_DATA_UINT64 }, 265*0Sstevel@tonic-gate { "intrthread", KSTAT_DATA_UINT64 }, 266*0Sstevel@tonic-gate { "intrblk", KSTAT_DATA_UINT64 }, 267*0Sstevel@tonic-gate { "intrunpin", KSTAT_DATA_UINT64 }, 268*0Sstevel@tonic-gate { "idlethread", KSTAT_DATA_UINT64 }, 269*0Sstevel@tonic-gate { "inv_swtch", KSTAT_DATA_UINT64 }, 270*0Sstevel@tonic-gate { "nthreads", KSTAT_DATA_UINT64 }, 271*0Sstevel@tonic-gate { "cpumigrate", KSTAT_DATA_UINT64 }, 272*0Sstevel@tonic-gate { "xcalls", KSTAT_DATA_UINT64 }, 273*0Sstevel@tonic-gate { "mutex_adenters", KSTAT_DATA_UINT64 }, 274*0Sstevel@tonic-gate { "rw_rdfails", KSTAT_DATA_UINT64 }, 275*0Sstevel@tonic-gate { "rw_wrfails", KSTAT_DATA_UINT64 }, 276*0Sstevel@tonic-gate { "modload", KSTAT_DATA_UINT64 }, 277*0Sstevel@tonic-gate { "modunload", KSTAT_DATA_UINT64 }, 278*0Sstevel@tonic-gate { "bawrite", KSTAT_DATA_UINT64 }, 279*0Sstevel@tonic-gate { "iowait", KSTAT_DATA_UINT64 }, 280*0Sstevel@tonic-gate }; 281*0Sstevel@tonic-gate 282*0Sstevel@tonic-gate static struct cpu_vm_stats_ks_data { 283*0Sstevel@tonic-gate kstat_named_t pgrec; 284*0Sstevel@tonic-gate kstat_named_t pgfrec; 285*0Sstevel@tonic-gate kstat_named_t pgin; 286*0Sstevel@tonic-gate kstat_named_t pgpgin; 287*0Sstevel@tonic-gate kstat_named_t pgout; 288*0Sstevel@tonic-gate kstat_named_t pgpgout; 289*0Sstevel@tonic-gate kstat_named_t swapin; 290*0Sstevel@tonic-gate kstat_named_t pgswapin; 291*0Sstevel@tonic-gate kstat_named_t swapout; 292*0Sstevel@tonic-gate kstat_named_t pgswapout; 293*0Sstevel@tonic-gate kstat_named_t zfod; 294*0Sstevel@tonic-gate kstat_named_t dfree; 295*0Sstevel@tonic-gate kstat_named_t scan; 296*0Sstevel@tonic-gate kstat_named_t rev; 297*0Sstevel@tonic-gate kstat_named_t hat_fault; 298*0Sstevel@tonic-gate kstat_named_t as_fault; 299*0Sstevel@tonic-gate kstat_named_t maj_fault; 300*0Sstevel@tonic-gate kstat_named_t cow_fault; 301*0Sstevel@tonic-gate kstat_named_t prot_fault; 302*0Sstevel@tonic-gate kstat_named_t softlock; 303*0Sstevel@tonic-gate kstat_named_t kernel_asflt; 304*0Sstevel@tonic-gate kstat_named_t pgrrun; 305*0Sstevel@tonic-gate kstat_named_t execpgin; 306*0Sstevel@tonic-gate kstat_named_t execpgout; 307*0Sstevel@tonic-gate kstat_named_t execfree; 308*0Sstevel@tonic-gate kstat_named_t anonpgin; 309*0Sstevel@tonic-gate kstat_named_t anonpgout; 310*0Sstevel@tonic-gate kstat_named_t anonfree; 311*0Sstevel@tonic-gate kstat_named_t fspgin; 312*0Sstevel@tonic-gate kstat_named_t fspgout; 313*0Sstevel@tonic-gate kstat_named_t fsfree; 314*0Sstevel@tonic-gate } cpu_vm_stats_ks_data_template = { 315*0Sstevel@tonic-gate { "pgrec", KSTAT_DATA_UINT64 }, 316*0Sstevel@tonic-gate { "pgfrec", KSTAT_DATA_UINT64 }, 317*0Sstevel@tonic-gate { "pgin", KSTAT_DATA_UINT64 }, 318*0Sstevel@tonic-gate { "pgpgin", KSTAT_DATA_UINT64 }, 319*0Sstevel@tonic-gate { "pgout", KSTAT_DATA_UINT64 }, 320*0Sstevel@tonic-gate { "pgpgout", KSTAT_DATA_UINT64 }, 321*0Sstevel@tonic-gate { "swapin", KSTAT_DATA_UINT64 }, 322*0Sstevel@tonic-gate { "pgswapin", KSTAT_DATA_UINT64 }, 323*0Sstevel@tonic-gate { "swapout", KSTAT_DATA_UINT64 }, 324*0Sstevel@tonic-gate { "pgswapout", KSTAT_DATA_UINT64 }, 325*0Sstevel@tonic-gate { "zfod", KSTAT_DATA_UINT64 }, 326*0Sstevel@tonic-gate { "dfree", KSTAT_DATA_UINT64 }, 327*0Sstevel@tonic-gate { "scan", KSTAT_DATA_UINT64 }, 328*0Sstevel@tonic-gate { "rev", KSTAT_DATA_UINT64 }, 329*0Sstevel@tonic-gate { "hat_fault", KSTAT_DATA_UINT64 }, 330*0Sstevel@tonic-gate { "as_fault", KSTAT_DATA_UINT64 }, 331*0Sstevel@tonic-gate { "maj_fault", KSTAT_DATA_UINT64 }, 332*0Sstevel@tonic-gate { "cow_fault", KSTAT_DATA_UINT64 }, 333*0Sstevel@tonic-gate { "prot_fault", KSTAT_DATA_UINT64 }, 334*0Sstevel@tonic-gate { "softlock", KSTAT_DATA_UINT64 }, 335*0Sstevel@tonic-gate { "kernel_asflt", KSTAT_DATA_UINT64 }, 336*0Sstevel@tonic-gate { "pgrrun", KSTAT_DATA_UINT64 }, 337*0Sstevel@tonic-gate { "execpgin", KSTAT_DATA_UINT64 }, 338*0Sstevel@tonic-gate { "execpgout", KSTAT_DATA_UINT64 }, 339*0Sstevel@tonic-gate { "execfree", KSTAT_DATA_UINT64 }, 340*0Sstevel@tonic-gate { "anonpgin", KSTAT_DATA_UINT64 }, 341*0Sstevel@tonic-gate { "anonpgout", KSTAT_DATA_UINT64 }, 342*0Sstevel@tonic-gate { "anonfree", KSTAT_DATA_UINT64 }, 343*0Sstevel@tonic-gate { "fspgin", KSTAT_DATA_UINT64 }, 344*0Sstevel@tonic-gate { "fspgout", KSTAT_DATA_UINT64 }, 345*0Sstevel@tonic-gate { "fsfree", KSTAT_DATA_UINT64 }, 346*0Sstevel@tonic-gate }; 347*0Sstevel@tonic-gate 348*0Sstevel@tonic-gate /* 349*0Sstevel@tonic-gate * Force the specified thread to migrate to the appropriate processor. 350*0Sstevel@tonic-gate * Called with thread lock held, returns with it dropped. 351*0Sstevel@tonic-gate */ 352*0Sstevel@tonic-gate static void 353*0Sstevel@tonic-gate force_thread_migrate(kthread_id_t tp) 354*0Sstevel@tonic-gate { 355*0Sstevel@tonic-gate ASSERT(THREAD_LOCK_HELD(tp)); 356*0Sstevel@tonic-gate if (tp == curthread) { 357*0Sstevel@tonic-gate THREAD_TRANSITION(tp); 358*0Sstevel@tonic-gate CL_SETRUN(tp); 359*0Sstevel@tonic-gate thread_unlock_nopreempt(tp); 360*0Sstevel@tonic-gate swtch(); 361*0Sstevel@tonic-gate } else { 362*0Sstevel@tonic-gate if (tp->t_state == TS_ONPROC) { 363*0Sstevel@tonic-gate cpu_surrender(tp); 364*0Sstevel@tonic-gate } else if (tp->t_state == TS_RUN) { 365*0Sstevel@tonic-gate (void) dispdeq(tp); 366*0Sstevel@tonic-gate setbackdq(tp); 367*0Sstevel@tonic-gate } 368*0Sstevel@tonic-gate thread_unlock(tp); 369*0Sstevel@tonic-gate } 370*0Sstevel@tonic-gate } 371*0Sstevel@tonic-gate 372*0Sstevel@tonic-gate /* 373*0Sstevel@tonic-gate * Set affinity for a specified CPU. 374*0Sstevel@tonic-gate * A reference count is incremented and the affinity is held until the 375*0Sstevel@tonic-gate * reference count is decremented to zero by thread_affinity_clear(). 376*0Sstevel@tonic-gate * This is so regions of code requiring affinity can be nested. 377*0Sstevel@tonic-gate * Caller needs to ensure that cpu_id remains valid, which can be 378*0Sstevel@tonic-gate * done by holding cpu_lock across this call, unless the caller 379*0Sstevel@tonic-gate * specifies CPU_CURRENT in which case the cpu_lock will be acquired 380*0Sstevel@tonic-gate * by thread_affinity_set and CPU->cpu_id will be the target CPU. 381*0Sstevel@tonic-gate */ 382*0Sstevel@tonic-gate void 383*0Sstevel@tonic-gate thread_affinity_set(kthread_id_t t, int cpu_id) 384*0Sstevel@tonic-gate { 385*0Sstevel@tonic-gate cpu_t *cp; 386*0Sstevel@tonic-gate int c; 387*0Sstevel@tonic-gate 388*0Sstevel@tonic-gate ASSERT(!(t == curthread && t->t_weakbound_cpu != NULL)); 389*0Sstevel@tonic-gate 390*0Sstevel@tonic-gate if ((c = cpu_id) == CPU_CURRENT) { 391*0Sstevel@tonic-gate mutex_enter(&cpu_lock); 392*0Sstevel@tonic-gate cpu_id = CPU->cpu_id; 393*0Sstevel@tonic-gate } 394*0Sstevel@tonic-gate /* 395*0Sstevel@tonic-gate * We should be asserting that cpu_lock is held here, but 396*0Sstevel@tonic-gate * the NCA code doesn't acquire it. The following assert 397*0Sstevel@tonic-gate * should be uncommented when the NCA code is fixed. 398*0Sstevel@tonic-gate * 399*0Sstevel@tonic-gate * ASSERT(MUTEX_HELD(&cpu_lock)); 400*0Sstevel@tonic-gate */ 401*0Sstevel@tonic-gate ASSERT((cpu_id >= 0) && (cpu_id < NCPU)); 402*0Sstevel@tonic-gate cp = cpu[cpu_id]; 403*0Sstevel@tonic-gate ASSERT(cp != NULL); /* user must provide a good cpu_id */ 404*0Sstevel@tonic-gate /* 405*0Sstevel@tonic-gate * If there is already a hard affinity requested, and this affinity 406*0Sstevel@tonic-gate * conflicts with that, panic. 407*0Sstevel@tonic-gate */ 408*0Sstevel@tonic-gate thread_lock(t); 409*0Sstevel@tonic-gate if (t->t_affinitycnt > 0 && t->t_bound_cpu != cp) { 410*0Sstevel@tonic-gate panic("affinity_set: setting %p but already bound to %p", 411*0Sstevel@tonic-gate (void *)cp, (void *)t->t_bound_cpu); 412*0Sstevel@tonic-gate } 413*0Sstevel@tonic-gate t->t_affinitycnt++; 414*0Sstevel@tonic-gate t->t_bound_cpu = cp; 415*0Sstevel@tonic-gate 416*0Sstevel@tonic-gate /* 417*0Sstevel@tonic-gate * Make sure we're running on the right CPU. 418*0Sstevel@tonic-gate */ 419*0Sstevel@tonic-gate if (cp != t->t_cpu || t != curthread) { 420*0Sstevel@tonic-gate force_thread_migrate(t); /* drops thread lock */ 421*0Sstevel@tonic-gate } else { 422*0Sstevel@tonic-gate thread_unlock(t); 423*0Sstevel@tonic-gate } 424*0Sstevel@tonic-gate 425*0Sstevel@tonic-gate if (c == CPU_CURRENT) 426*0Sstevel@tonic-gate mutex_exit(&cpu_lock); 427*0Sstevel@tonic-gate } 428*0Sstevel@tonic-gate 429*0Sstevel@tonic-gate /* 430*0Sstevel@tonic-gate * Wrapper for backward compatibility. 431*0Sstevel@tonic-gate */ 432*0Sstevel@tonic-gate void 433*0Sstevel@tonic-gate affinity_set(int cpu_id) 434*0Sstevel@tonic-gate { 435*0Sstevel@tonic-gate thread_affinity_set(curthread, cpu_id); 436*0Sstevel@tonic-gate } 437*0Sstevel@tonic-gate 438*0Sstevel@tonic-gate /* 439*0Sstevel@tonic-gate * Decrement the affinity reservation count and if it becomes zero, 440*0Sstevel@tonic-gate * clear the CPU affinity for the current thread, or set it to the user's 441*0Sstevel@tonic-gate * software binding request. 442*0Sstevel@tonic-gate */ 443*0Sstevel@tonic-gate void 444*0Sstevel@tonic-gate thread_affinity_clear(kthread_id_t t) 445*0Sstevel@tonic-gate { 446*0Sstevel@tonic-gate register processorid_t binding; 447*0Sstevel@tonic-gate 448*0Sstevel@tonic-gate thread_lock(t); 449*0Sstevel@tonic-gate if (--t->t_affinitycnt == 0) { 450*0Sstevel@tonic-gate if ((binding = t->t_bind_cpu) == PBIND_NONE) { 451*0Sstevel@tonic-gate /* 452*0Sstevel@tonic-gate * Adjust disp_max_unbound_pri if necessary. 453*0Sstevel@tonic-gate */ 454*0Sstevel@tonic-gate disp_adjust_unbound_pri(t); 455*0Sstevel@tonic-gate t->t_bound_cpu = NULL; 456*0Sstevel@tonic-gate if (t->t_cpu->cpu_part != t->t_cpupart) { 457*0Sstevel@tonic-gate force_thread_migrate(t); 458*0Sstevel@tonic-gate return; 459*0Sstevel@tonic-gate } 460*0Sstevel@tonic-gate } else { 461*0Sstevel@tonic-gate t->t_bound_cpu = cpu[binding]; 462*0Sstevel@tonic-gate /* 463*0Sstevel@tonic-gate * Make sure the thread is running on the bound CPU. 464*0Sstevel@tonic-gate */ 465*0Sstevel@tonic-gate if (t->t_cpu != t->t_bound_cpu) { 466*0Sstevel@tonic-gate force_thread_migrate(t); 467*0Sstevel@tonic-gate return; /* already dropped lock */ 468*0Sstevel@tonic-gate } 469*0Sstevel@tonic-gate } 470*0Sstevel@tonic-gate } 471*0Sstevel@tonic-gate thread_unlock(t); 472*0Sstevel@tonic-gate } 473*0Sstevel@tonic-gate 474*0Sstevel@tonic-gate /* 475*0Sstevel@tonic-gate * Wrapper for backward compatibility. 476*0Sstevel@tonic-gate */ 477*0Sstevel@tonic-gate void 478*0Sstevel@tonic-gate affinity_clear(void) 479*0Sstevel@tonic-gate { 480*0Sstevel@tonic-gate thread_affinity_clear(curthread); 481*0Sstevel@tonic-gate } 482*0Sstevel@tonic-gate 483*0Sstevel@tonic-gate /* 484*0Sstevel@tonic-gate * Weak cpu affinity. Bind to the "current" cpu for short periods 485*0Sstevel@tonic-gate * of time during which the thread must not block (but may be preempted). 486*0Sstevel@tonic-gate * Use this instead of kpreempt_disable() when it is only "no migration" 487*0Sstevel@tonic-gate * rather than "no preemption" semantics that are required - disabling 488*0Sstevel@tonic-gate * preemption holds higher priority threads off of cpu and if the 489*0Sstevel@tonic-gate * operation that is protected is more than momentary this is not good 490*0Sstevel@tonic-gate * for realtime etc. 491*0Sstevel@tonic-gate * 492*0Sstevel@tonic-gate * Weakly bound threads will not prevent a cpu from being offlined - 493*0Sstevel@tonic-gate * we'll only run them on the cpu to which they are weakly bound but 494*0Sstevel@tonic-gate * (because they do not block) we'll always be able to move them on to 495*0Sstevel@tonic-gate * another cpu at offline time if we give them just a short moment to 496*0Sstevel@tonic-gate * run during which they will unbind. To give a cpu a chance of offlining, 497*0Sstevel@tonic-gate * however, we require a barrier to weak bindings that may be raised for a 498*0Sstevel@tonic-gate * given cpu (offline/move code may set this and then wait a short time for 499*0Sstevel@tonic-gate * existing weak bindings to drop); the cpu_inmotion pointer is that barrier. 500*0Sstevel@tonic-gate * 501*0Sstevel@tonic-gate * There are few restrictions on the calling context of thread_nomigrate. 502*0Sstevel@tonic-gate * The caller must not hold the thread lock. Calls may be nested. 503*0Sstevel@tonic-gate * 504*0Sstevel@tonic-gate * After weakbinding a thread must not perform actions that may block. 505*0Sstevel@tonic-gate * In particular it must not call thread_affinity_set; calling that when 506*0Sstevel@tonic-gate * already weakbound is nonsensical anyway. 507*0Sstevel@tonic-gate * 508*0Sstevel@tonic-gate * If curthread is prevented from migrating for other reasons 509*0Sstevel@tonic-gate * (kernel preemption disabled; high pil; strongly bound; interrupt thread) 510*0Sstevel@tonic-gate * then the weak binding will succeed even if this cpu is the target of an 511*0Sstevel@tonic-gate * offline/move request. 512*0Sstevel@tonic-gate */ 513*0Sstevel@tonic-gate void 514*0Sstevel@tonic-gate thread_nomigrate(void) 515*0Sstevel@tonic-gate { 516*0Sstevel@tonic-gate cpu_t *cp; 517*0Sstevel@tonic-gate kthread_id_t t = curthread; 518*0Sstevel@tonic-gate 519*0Sstevel@tonic-gate again: 520*0Sstevel@tonic-gate kpreempt_disable(); 521*0Sstevel@tonic-gate cp = CPU; 522*0Sstevel@tonic-gate 523*0Sstevel@tonic-gate /* 524*0Sstevel@tonic-gate * A highlevel interrupt must not modify t_nomigrate or 525*0Sstevel@tonic-gate * t_weakbound_cpu of the thread it has interrupted. A lowlevel 526*0Sstevel@tonic-gate * interrupt thread cannot migrate and we can avoid the 527*0Sstevel@tonic-gate * thread_lock call below by short-circuiting here. In either 528*0Sstevel@tonic-gate * case we can just return since no migration is possible and 529*0Sstevel@tonic-gate * the condition will persist (ie, when we test for these again 530*0Sstevel@tonic-gate * in thread_allowmigrate they can't have changed). Migration 531*0Sstevel@tonic-gate * is also impossible if we're at or above DISP_LEVEL pil. 532*0Sstevel@tonic-gate */ 533*0Sstevel@tonic-gate if (CPU_ON_INTR(cp) || t->t_flag & T_INTR_THREAD || 534*0Sstevel@tonic-gate getpil() >= DISP_LEVEL) { 535*0Sstevel@tonic-gate kpreempt_enable(); 536*0Sstevel@tonic-gate return; 537*0Sstevel@tonic-gate } 538*0Sstevel@tonic-gate 539*0Sstevel@tonic-gate /* 540*0Sstevel@tonic-gate * We must be consistent with existing weak bindings. Since we 541*0Sstevel@tonic-gate * may be interrupted between the increment of t_nomigrate and 542*0Sstevel@tonic-gate * the store to t_weakbound_cpu below we cannot assume that 543*0Sstevel@tonic-gate * t_weakbound_cpu will be set if t_nomigrate is. Note that we 544*0Sstevel@tonic-gate * cannot assert t_weakbound_cpu == t_bind_cpu since that is not 545*0Sstevel@tonic-gate * always the case. 546*0Sstevel@tonic-gate */ 547*0Sstevel@tonic-gate if (t->t_nomigrate && t->t_weakbound_cpu && t->t_weakbound_cpu != cp) { 548*0Sstevel@tonic-gate if (!panicstr) 549*0Sstevel@tonic-gate panic("thread_nomigrate: binding to %p but already " 550*0Sstevel@tonic-gate "bound to %p", (void *)cp, 551*0Sstevel@tonic-gate (void *)t->t_weakbound_cpu); 552*0Sstevel@tonic-gate } 553*0Sstevel@tonic-gate 554*0Sstevel@tonic-gate /* 555*0Sstevel@tonic-gate * At this point we have preemption disabled and we don't yet hold 556*0Sstevel@tonic-gate * the thread lock. So it's possible that somebody else could 557*0Sstevel@tonic-gate * set t_bind_cpu here and not be able to force us across to the 558*0Sstevel@tonic-gate * new cpu (since we have preemption disabled). 559*0Sstevel@tonic-gate */ 560*0Sstevel@tonic-gate thread_lock(curthread); 561*0Sstevel@tonic-gate 562*0Sstevel@tonic-gate /* 563*0Sstevel@tonic-gate * If further weak bindings are being (temporarily) suppressed then 564*0Sstevel@tonic-gate * we'll settle for disabling kernel preemption (which assures 565*0Sstevel@tonic-gate * no migration provided the thread does not block which it is 566*0Sstevel@tonic-gate * not allowed to if using thread_nomigrate). We must remember 567*0Sstevel@tonic-gate * this disposition so we can take appropriate action in 568*0Sstevel@tonic-gate * thread_allowmigrate. If this is a nested call and the 569*0Sstevel@tonic-gate * thread is already weakbound then fall through as normal. 570*0Sstevel@tonic-gate * We remember the decision to settle for kpreempt_disable through 571*0Sstevel@tonic-gate * negative nesting counting in t_nomigrate. Once a thread has had one 572*0Sstevel@tonic-gate * weakbinding request satisfied in this way any further (nested) 573*0Sstevel@tonic-gate * requests will continue to be satisfied in the same way, 574*0Sstevel@tonic-gate * even if weak bindings have recommenced. 575*0Sstevel@tonic-gate */ 576*0Sstevel@tonic-gate if (t->t_nomigrate < 0 || weakbindingbarrier && t->t_nomigrate == 0) { 577*0Sstevel@tonic-gate --t->t_nomigrate; 578*0Sstevel@tonic-gate thread_unlock(curthread); 579*0Sstevel@tonic-gate return; /* with kpreempt_disable still active */ 580*0Sstevel@tonic-gate } 581*0Sstevel@tonic-gate 582*0Sstevel@tonic-gate /* 583*0Sstevel@tonic-gate * We hold thread_lock so t_bind_cpu cannot change. We could, 584*0Sstevel@tonic-gate * however, be running on a different cpu to which we are t_bound_cpu 585*0Sstevel@tonic-gate * to (as explained above). If we grant the weak binding request 586*0Sstevel@tonic-gate * in that case then the dispatcher must favour our weak binding 587*0Sstevel@tonic-gate * over our strong (in which case, just as when preemption is 588*0Sstevel@tonic-gate * disabled, we can continue to run on a cpu other than the one to 589*0Sstevel@tonic-gate * which we are strongbound; the difference in this case is that 590*0Sstevel@tonic-gate * this thread can be preempted and so can appear on the dispatch 591*0Sstevel@tonic-gate * queues of a cpu other than the one it is strongbound to). 592*0Sstevel@tonic-gate * 593*0Sstevel@tonic-gate * If the cpu we are running on does not appear to be a current 594*0Sstevel@tonic-gate * offline target (we check cpu_inmotion to determine this - since 595*0Sstevel@tonic-gate * we don't hold cpu_lock we may not see a recent store to that, 596*0Sstevel@tonic-gate * so it's possible that we at times can grant a weak binding to a 597*0Sstevel@tonic-gate * cpu that is an offline target, but that one request will not 598*0Sstevel@tonic-gate * prevent the offline from succeeding) then we will always grant 599*0Sstevel@tonic-gate * the weak binding request. This includes the case above where 600*0Sstevel@tonic-gate * we grant a weakbinding not commensurate with our strong binding. 601*0Sstevel@tonic-gate * 602*0Sstevel@tonic-gate * If our cpu does appear to be an offline target then we're inclined 603*0Sstevel@tonic-gate * not to grant the weakbinding request just yet - we'd prefer to 604*0Sstevel@tonic-gate * migrate to another cpu and grant the request there. The 605*0Sstevel@tonic-gate * exceptions are those cases where going through preemption code 606*0Sstevel@tonic-gate * will not result in us changing cpu: 607*0Sstevel@tonic-gate * 608*0Sstevel@tonic-gate * . interrupts have already bypassed this case (see above) 609*0Sstevel@tonic-gate * . we are already weakbound to this cpu (dispatcher code will 610*0Sstevel@tonic-gate * always return us to the weakbound cpu) 611*0Sstevel@tonic-gate * . preemption was disabled even before we disabled it above 612*0Sstevel@tonic-gate * . we are strongbound to this cpu (if we're strongbound to 613*0Sstevel@tonic-gate * another and not yet running there the trip through the 614*0Sstevel@tonic-gate * dispatcher will move us to the strongbound cpu and we 615*0Sstevel@tonic-gate * will grant the weak binding there) 616*0Sstevel@tonic-gate */ 617*0Sstevel@tonic-gate if (cp != cpu_inmotion || t->t_nomigrate > 0 || t->t_preempt > 1 || 618*0Sstevel@tonic-gate t->t_bound_cpu == cp) { 619*0Sstevel@tonic-gate /* 620*0Sstevel@tonic-gate * Don't be tempted to store to t_weakbound_cpu only on 621*0Sstevel@tonic-gate * the first nested bind request - if we're interrupted 622*0Sstevel@tonic-gate * after the increment of t_nomigrate and before the 623*0Sstevel@tonic-gate * store to t_weakbound_cpu and the interrupt calls 624*0Sstevel@tonic-gate * thread_nomigrate then the assertion in thread_allowmigrate 625*0Sstevel@tonic-gate * would fail. 626*0Sstevel@tonic-gate */ 627*0Sstevel@tonic-gate t->t_nomigrate++; 628*0Sstevel@tonic-gate t->t_weakbound_cpu = cp; 629*0Sstevel@tonic-gate membar_producer(); 630*0Sstevel@tonic-gate thread_unlock(curthread); 631*0Sstevel@tonic-gate /* 632*0Sstevel@tonic-gate * Now that we have dropped the thread_lock another thread 633*0Sstevel@tonic-gate * can set our t_weakbound_cpu, and will try to migrate us 634*0Sstevel@tonic-gate * to the strongbound cpu (which will not be prevented by 635*0Sstevel@tonic-gate * preemption being disabled since we're about to enable 636*0Sstevel@tonic-gate * preemption). We have granted the weakbinding to the current 637*0Sstevel@tonic-gate * cpu, so again we are in the position that is is is possible 638*0Sstevel@tonic-gate * that our weak and strong bindings differ. Again this 639*0Sstevel@tonic-gate * is catered for by dispatcher code which will favour our 640*0Sstevel@tonic-gate * weak binding. 641*0Sstevel@tonic-gate */ 642*0Sstevel@tonic-gate kpreempt_enable(); 643*0Sstevel@tonic-gate } else { 644*0Sstevel@tonic-gate /* 645*0Sstevel@tonic-gate * Move to another cpu before granting the request by 646*0Sstevel@tonic-gate * forcing this thread through preemption code. When we 647*0Sstevel@tonic-gate * get to set{front,back}dq called from CL_PREEMPT() 648*0Sstevel@tonic-gate * cpu_choose() will be used to select a cpu to queue 649*0Sstevel@tonic-gate * us on - that will see cpu_inmotion and take 650*0Sstevel@tonic-gate * steps to avoid returning us to this cpu. 651*0Sstevel@tonic-gate */ 652*0Sstevel@tonic-gate cp->cpu_kprunrun = 1; 653*0Sstevel@tonic-gate thread_unlock(curthread); 654*0Sstevel@tonic-gate kpreempt_enable(); /* will call preempt() */ 655*0Sstevel@tonic-gate goto again; 656*0Sstevel@tonic-gate } 657*0Sstevel@tonic-gate } 658*0Sstevel@tonic-gate 659*0Sstevel@tonic-gate void 660*0Sstevel@tonic-gate thread_allowmigrate(void) 661*0Sstevel@tonic-gate { 662*0Sstevel@tonic-gate kthread_id_t t = curthread; 663*0Sstevel@tonic-gate 664*0Sstevel@tonic-gate ASSERT(t->t_weakbound_cpu == CPU || 665*0Sstevel@tonic-gate (t->t_nomigrate < 0 && t->t_preempt > 0) || 666*0Sstevel@tonic-gate CPU_ON_INTR(CPU) || t->t_flag & T_INTR_THREAD || 667*0Sstevel@tonic-gate getpil() >= DISP_LEVEL); 668*0Sstevel@tonic-gate 669*0Sstevel@tonic-gate if (CPU_ON_INTR(CPU) || (t->t_flag & T_INTR_THREAD) || 670*0Sstevel@tonic-gate getpil() >= DISP_LEVEL) 671*0Sstevel@tonic-gate return; 672*0Sstevel@tonic-gate 673*0Sstevel@tonic-gate if (t->t_nomigrate < 0) { 674*0Sstevel@tonic-gate /* 675*0Sstevel@tonic-gate * This thread was granted "weak binding" in the 676*0Sstevel@tonic-gate * stronger form of kernel preemption disabling. 677*0Sstevel@tonic-gate * Undo a level of nesting for both t_nomigrate 678*0Sstevel@tonic-gate * and t_preempt. 679*0Sstevel@tonic-gate */ 680*0Sstevel@tonic-gate ++t->t_nomigrate; 681*0Sstevel@tonic-gate kpreempt_enable(); 682*0Sstevel@tonic-gate } else if (--t->t_nomigrate == 0) { 683*0Sstevel@tonic-gate /* 684*0Sstevel@tonic-gate * Time to drop the weak binding. We need to cater 685*0Sstevel@tonic-gate * for the case where we're weakbound to a different 686*0Sstevel@tonic-gate * cpu than that to which we're strongbound (a very 687*0Sstevel@tonic-gate * temporary arrangement that must only persist until 688*0Sstevel@tonic-gate * weak binding drops). We don't acquire thread_lock 689*0Sstevel@tonic-gate * here so even as this code executes t_bound_cpu 690*0Sstevel@tonic-gate * may be changing. So we disable preemption and 691*0Sstevel@tonic-gate * a) in the case that t_bound_cpu changes while we 692*0Sstevel@tonic-gate * have preemption disabled kprunrun will be set 693*0Sstevel@tonic-gate * asynchronously, and b) if before disabling 694*0Sstevel@tonic-gate * preemption we were already on a different cpu to 695*0Sstevel@tonic-gate * our t_bound_cpu then we set kprunrun ourselves 696*0Sstevel@tonic-gate * to force a trip through the dispatcher when 697*0Sstevel@tonic-gate * preemption is enabled. 698*0Sstevel@tonic-gate */ 699*0Sstevel@tonic-gate kpreempt_disable(); 700*0Sstevel@tonic-gate if (t->t_bound_cpu && 701*0Sstevel@tonic-gate t->t_weakbound_cpu != t->t_bound_cpu) 702*0Sstevel@tonic-gate CPU->cpu_kprunrun = 1; 703*0Sstevel@tonic-gate t->t_weakbound_cpu = NULL; 704*0Sstevel@tonic-gate membar_producer(); 705*0Sstevel@tonic-gate kpreempt_enable(); 706*0Sstevel@tonic-gate } 707*0Sstevel@tonic-gate } 708*0Sstevel@tonic-gate 709*0Sstevel@tonic-gate /* 710*0Sstevel@tonic-gate * weakbinding_stop can be used to temporarily cause weakbindings made 711*0Sstevel@tonic-gate * with thread_nomigrate to be satisfied through the stronger action of 712*0Sstevel@tonic-gate * kpreempt_disable. weakbinding_start recommences normal weakbinding. 713*0Sstevel@tonic-gate */ 714*0Sstevel@tonic-gate 715*0Sstevel@tonic-gate void 716*0Sstevel@tonic-gate weakbinding_stop(void) 717*0Sstevel@tonic-gate { 718*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 719*0Sstevel@tonic-gate weakbindingbarrier = 1; 720*0Sstevel@tonic-gate membar_producer(); /* make visible before subsequent thread_lock */ 721*0Sstevel@tonic-gate } 722*0Sstevel@tonic-gate 723*0Sstevel@tonic-gate void 724*0Sstevel@tonic-gate weakbinding_start(void) 725*0Sstevel@tonic-gate { 726*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 727*0Sstevel@tonic-gate weakbindingbarrier = 0; 728*0Sstevel@tonic-gate } 729*0Sstevel@tonic-gate 730*0Sstevel@tonic-gate /* 731*0Sstevel@tonic-gate * This routine is called to place the CPUs in a safe place so that 732*0Sstevel@tonic-gate * one of them can be taken off line or placed on line. What we are 733*0Sstevel@tonic-gate * trying to do here is prevent a thread from traversing the list 734*0Sstevel@tonic-gate * of active CPUs while we are changing it or from getting placed on 735*0Sstevel@tonic-gate * the run queue of a CPU that has just gone off line. We do this by 736*0Sstevel@tonic-gate * creating a thread with the highest possible prio for each CPU and 737*0Sstevel@tonic-gate * having it call this routine. The advantage of this method is that 738*0Sstevel@tonic-gate * we can eliminate all checks for CPU_ACTIVE in the disp routines. 739*0Sstevel@tonic-gate * This makes disp faster at the expense of making p_online() slower 740*0Sstevel@tonic-gate * which is a good trade off. 741*0Sstevel@tonic-gate */ 742*0Sstevel@tonic-gate static void 743*0Sstevel@tonic-gate cpu_pause(volatile char *safe) 744*0Sstevel@tonic-gate { 745*0Sstevel@tonic-gate int s; 746*0Sstevel@tonic-gate struct _cpu_pause_info *cpi = &cpu_pause_info; 747*0Sstevel@tonic-gate 748*0Sstevel@tonic-gate ASSERT((curthread->t_bound_cpu != NULL) || (*safe == PAUSE_DIE)); 749*0Sstevel@tonic-gate 750*0Sstevel@tonic-gate while (*safe != PAUSE_DIE) { 751*0Sstevel@tonic-gate *safe = PAUSE_READY; 752*0Sstevel@tonic-gate membar_enter(); /* make sure stores are flushed */ 753*0Sstevel@tonic-gate sema_v(&cpi->cp_sem); /* signal requesting thread */ 754*0Sstevel@tonic-gate 755*0Sstevel@tonic-gate /* 756*0Sstevel@tonic-gate * Wait here until all pause threads are running. That 757*0Sstevel@tonic-gate * indicates that it's safe to do the spl. Until 758*0Sstevel@tonic-gate * cpu_pause_info.cp_go is set, we don't want to spl 759*0Sstevel@tonic-gate * because that might block clock interrupts needed 760*0Sstevel@tonic-gate * to preempt threads on other CPUs. 761*0Sstevel@tonic-gate */ 762*0Sstevel@tonic-gate while (cpi->cp_go == 0) 763*0Sstevel@tonic-gate ; 764*0Sstevel@tonic-gate /* 765*0Sstevel@tonic-gate * Even though we are at the highest disp prio, we need 766*0Sstevel@tonic-gate * to block out all interrupts below LOCK_LEVEL so that 767*0Sstevel@tonic-gate * an intr doesn't come in, wake up a thread, and call 768*0Sstevel@tonic-gate * setbackdq/setfrontdq. 769*0Sstevel@tonic-gate */ 770*0Sstevel@tonic-gate s = splhigh(); 771*0Sstevel@tonic-gate /* 772*0Sstevel@tonic-gate * This cpu is now safe. 773*0Sstevel@tonic-gate */ 774*0Sstevel@tonic-gate *safe = PAUSE_WAIT; 775*0Sstevel@tonic-gate membar_enter(); /* make sure stores are flushed */ 776*0Sstevel@tonic-gate 777*0Sstevel@tonic-gate /* 778*0Sstevel@tonic-gate * Now we wait. When we are allowed to continue, safe will 779*0Sstevel@tonic-gate * be set to PAUSE_IDLE. 780*0Sstevel@tonic-gate */ 781*0Sstevel@tonic-gate while (*safe != PAUSE_IDLE) 782*0Sstevel@tonic-gate ; 783*0Sstevel@tonic-gate 784*0Sstevel@tonic-gate splx(s); 785*0Sstevel@tonic-gate /* 786*0Sstevel@tonic-gate * Waiting is at an end. Switch out of cpu_pause 787*0Sstevel@tonic-gate * loop and resume useful work. 788*0Sstevel@tonic-gate */ 789*0Sstevel@tonic-gate swtch(); 790*0Sstevel@tonic-gate } 791*0Sstevel@tonic-gate 792*0Sstevel@tonic-gate mutex_enter(&pause_free_mutex); 793*0Sstevel@tonic-gate *safe = PAUSE_DEAD; 794*0Sstevel@tonic-gate cv_broadcast(&pause_free_cv); 795*0Sstevel@tonic-gate mutex_exit(&pause_free_mutex); 796*0Sstevel@tonic-gate } 797*0Sstevel@tonic-gate 798*0Sstevel@tonic-gate /* 799*0Sstevel@tonic-gate * Allow the cpus to start running again. 800*0Sstevel@tonic-gate */ 801*0Sstevel@tonic-gate void 802*0Sstevel@tonic-gate start_cpus() 803*0Sstevel@tonic-gate { 804*0Sstevel@tonic-gate int i; 805*0Sstevel@tonic-gate 806*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 807*0Sstevel@tonic-gate ASSERT(cpu_pause_info.cp_paused); 808*0Sstevel@tonic-gate cpu_pause_info.cp_paused = NULL; 809*0Sstevel@tonic-gate for (i = 0; i < NCPU; i++) 810*0Sstevel@tonic-gate safe_list[i] = PAUSE_IDLE; 811*0Sstevel@tonic-gate membar_enter(); /* make sure stores are flushed */ 812*0Sstevel@tonic-gate affinity_clear(); 813*0Sstevel@tonic-gate splx(cpu_pause_info.cp_spl); 814*0Sstevel@tonic-gate kpreempt_enable(); 815*0Sstevel@tonic-gate } 816*0Sstevel@tonic-gate 817*0Sstevel@tonic-gate /* 818*0Sstevel@tonic-gate * Allocate a pause thread for a CPU. 819*0Sstevel@tonic-gate */ 820*0Sstevel@tonic-gate static void 821*0Sstevel@tonic-gate cpu_pause_alloc(cpu_t *cp) 822*0Sstevel@tonic-gate { 823*0Sstevel@tonic-gate kthread_id_t t; 824*0Sstevel@tonic-gate int cpun = cp->cpu_id; 825*0Sstevel@tonic-gate 826*0Sstevel@tonic-gate /* 827*0Sstevel@tonic-gate * Note, v.v_nglobpris will not change value as long as I hold 828*0Sstevel@tonic-gate * cpu_lock. 829*0Sstevel@tonic-gate */ 830*0Sstevel@tonic-gate t = thread_create(NULL, 0, cpu_pause, (caddr_t)&safe_list[cpun], 831*0Sstevel@tonic-gate 0, &p0, TS_STOPPED, v.v_nglobpris - 1); 832*0Sstevel@tonic-gate thread_lock(t); 833*0Sstevel@tonic-gate t->t_bound_cpu = cp; 834*0Sstevel@tonic-gate t->t_disp_queue = cp->cpu_disp; 835*0Sstevel@tonic-gate t->t_affinitycnt = 1; 836*0Sstevel@tonic-gate t->t_preempt = 1; 837*0Sstevel@tonic-gate thread_unlock(t); 838*0Sstevel@tonic-gate cp->cpu_pause_thread = t; 839*0Sstevel@tonic-gate /* 840*0Sstevel@tonic-gate * Registering a thread in the callback table is usually done 841*0Sstevel@tonic-gate * in the initialization code of the thread. In this 842*0Sstevel@tonic-gate * case, we do it right after thread creation because the 843*0Sstevel@tonic-gate * thread itself may never run, and we need to register the 844*0Sstevel@tonic-gate * fact that it is safe for cpr suspend. 845*0Sstevel@tonic-gate */ 846*0Sstevel@tonic-gate CALLB_CPR_INIT_SAFE(t, "cpu_pause"); 847*0Sstevel@tonic-gate } 848*0Sstevel@tonic-gate 849*0Sstevel@tonic-gate /* 850*0Sstevel@tonic-gate * Free a pause thread for a CPU. 851*0Sstevel@tonic-gate */ 852*0Sstevel@tonic-gate static void 853*0Sstevel@tonic-gate cpu_pause_free(cpu_t *cp) 854*0Sstevel@tonic-gate { 855*0Sstevel@tonic-gate kthread_id_t t; 856*0Sstevel@tonic-gate int cpun = cp->cpu_id; 857*0Sstevel@tonic-gate 858*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 859*0Sstevel@tonic-gate /* 860*0Sstevel@tonic-gate * We have to get the thread and tell him to die. 861*0Sstevel@tonic-gate */ 862*0Sstevel@tonic-gate if ((t = cp->cpu_pause_thread) == NULL) { 863*0Sstevel@tonic-gate ASSERT(safe_list[cpun] == PAUSE_IDLE); 864*0Sstevel@tonic-gate return; 865*0Sstevel@tonic-gate } 866*0Sstevel@tonic-gate thread_lock(t); 867*0Sstevel@tonic-gate t->t_cpu = CPU; /* disp gets upset if last cpu is quiesced. */ 868*0Sstevel@tonic-gate t->t_bound_cpu = NULL; /* Must un-bind; cpu may not be running. */ 869*0Sstevel@tonic-gate t->t_pri = v.v_nglobpris - 1; 870*0Sstevel@tonic-gate ASSERT(safe_list[cpun] == PAUSE_IDLE); 871*0Sstevel@tonic-gate safe_list[cpun] = PAUSE_DIE; 872*0Sstevel@tonic-gate THREAD_TRANSITION(t); 873*0Sstevel@tonic-gate setbackdq(t); 874*0Sstevel@tonic-gate thread_unlock_nopreempt(t); 875*0Sstevel@tonic-gate 876*0Sstevel@tonic-gate /* 877*0Sstevel@tonic-gate * If we don't wait for the thread to actually die, it may try to 878*0Sstevel@tonic-gate * run on the wrong cpu as part of an actual call to pause_cpus(). 879*0Sstevel@tonic-gate */ 880*0Sstevel@tonic-gate mutex_enter(&pause_free_mutex); 881*0Sstevel@tonic-gate while (safe_list[cpun] != PAUSE_DEAD) { 882*0Sstevel@tonic-gate cv_wait(&pause_free_cv, &pause_free_mutex); 883*0Sstevel@tonic-gate } 884*0Sstevel@tonic-gate mutex_exit(&pause_free_mutex); 885*0Sstevel@tonic-gate safe_list[cpun] = PAUSE_IDLE; 886*0Sstevel@tonic-gate 887*0Sstevel@tonic-gate cp->cpu_pause_thread = NULL; 888*0Sstevel@tonic-gate } 889*0Sstevel@tonic-gate 890*0Sstevel@tonic-gate /* 891*0Sstevel@tonic-gate * Initialize basic structures for pausing CPUs. 892*0Sstevel@tonic-gate */ 893*0Sstevel@tonic-gate void 894*0Sstevel@tonic-gate cpu_pause_init() 895*0Sstevel@tonic-gate { 896*0Sstevel@tonic-gate sema_init(&cpu_pause_info.cp_sem, 0, NULL, SEMA_DEFAULT, NULL); 897*0Sstevel@tonic-gate /* 898*0Sstevel@tonic-gate * Create initial CPU pause thread. 899*0Sstevel@tonic-gate */ 900*0Sstevel@tonic-gate cpu_pause_alloc(CPU); 901*0Sstevel@tonic-gate } 902*0Sstevel@tonic-gate 903*0Sstevel@tonic-gate /* 904*0Sstevel@tonic-gate * Start the threads used to pause another CPU. 905*0Sstevel@tonic-gate */ 906*0Sstevel@tonic-gate static int 907*0Sstevel@tonic-gate cpu_pause_start(processorid_t cpu_id) 908*0Sstevel@tonic-gate { 909*0Sstevel@tonic-gate int i; 910*0Sstevel@tonic-gate int cpu_count = 0; 911*0Sstevel@tonic-gate 912*0Sstevel@tonic-gate for (i = 0; i < NCPU; i++) { 913*0Sstevel@tonic-gate cpu_t *cp; 914*0Sstevel@tonic-gate kthread_id_t t; 915*0Sstevel@tonic-gate 916*0Sstevel@tonic-gate cp = cpu[i]; 917*0Sstevel@tonic-gate if (!CPU_IN_SET(cpu_available, i) || (i == cpu_id)) { 918*0Sstevel@tonic-gate safe_list[i] = PAUSE_WAIT; 919*0Sstevel@tonic-gate continue; 920*0Sstevel@tonic-gate } 921*0Sstevel@tonic-gate 922*0Sstevel@tonic-gate /* 923*0Sstevel@tonic-gate * Skip CPU if it is quiesced or not yet started. 924*0Sstevel@tonic-gate */ 925*0Sstevel@tonic-gate if ((cp->cpu_flags & (CPU_QUIESCED | CPU_READY)) != CPU_READY) { 926*0Sstevel@tonic-gate safe_list[i] = PAUSE_WAIT; 927*0Sstevel@tonic-gate continue; 928*0Sstevel@tonic-gate } 929*0Sstevel@tonic-gate 930*0Sstevel@tonic-gate /* 931*0Sstevel@tonic-gate * Start this CPU's pause thread. 932*0Sstevel@tonic-gate */ 933*0Sstevel@tonic-gate t = cp->cpu_pause_thread; 934*0Sstevel@tonic-gate thread_lock(t); 935*0Sstevel@tonic-gate /* 936*0Sstevel@tonic-gate * Reset the priority, since nglobpris may have 937*0Sstevel@tonic-gate * changed since the thread was created, if someone 938*0Sstevel@tonic-gate * has loaded the RT (or some other) scheduling 939*0Sstevel@tonic-gate * class. 940*0Sstevel@tonic-gate */ 941*0Sstevel@tonic-gate t->t_pri = v.v_nglobpris - 1; 942*0Sstevel@tonic-gate THREAD_TRANSITION(t); 943*0Sstevel@tonic-gate setbackdq(t); 944*0Sstevel@tonic-gate thread_unlock_nopreempt(t); 945*0Sstevel@tonic-gate ++cpu_count; 946*0Sstevel@tonic-gate } 947*0Sstevel@tonic-gate return (cpu_count); 948*0Sstevel@tonic-gate } 949*0Sstevel@tonic-gate 950*0Sstevel@tonic-gate 951*0Sstevel@tonic-gate /* 952*0Sstevel@tonic-gate * Pause all of the CPUs except the one we are on by creating a high 953*0Sstevel@tonic-gate * priority thread bound to those CPUs. 954*0Sstevel@tonic-gate * 955*0Sstevel@tonic-gate * Note that one must be extremely careful regarding code 956*0Sstevel@tonic-gate * executed while CPUs are paused. Since a CPU may be paused 957*0Sstevel@tonic-gate * while a thread scheduling on that CPU is holding an adaptive 958*0Sstevel@tonic-gate * lock, code executed with CPUs paused must not acquire adaptive 959*0Sstevel@tonic-gate * (or low-level spin) locks. Also, such code must not block, 960*0Sstevel@tonic-gate * since the thread that is supposed to initiate the wakeup may 961*0Sstevel@tonic-gate * never run. 962*0Sstevel@tonic-gate * 963*0Sstevel@tonic-gate * With a few exceptions, the restrictions on code executed with CPUs 964*0Sstevel@tonic-gate * paused match those for code executed at high-level interrupt 965*0Sstevel@tonic-gate * context. 966*0Sstevel@tonic-gate */ 967*0Sstevel@tonic-gate void 968*0Sstevel@tonic-gate pause_cpus(cpu_t *off_cp) 969*0Sstevel@tonic-gate { 970*0Sstevel@tonic-gate processorid_t cpu_id; 971*0Sstevel@tonic-gate int i; 972*0Sstevel@tonic-gate struct _cpu_pause_info *cpi = &cpu_pause_info; 973*0Sstevel@tonic-gate 974*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 975*0Sstevel@tonic-gate ASSERT(cpi->cp_paused == NULL); 976*0Sstevel@tonic-gate cpi->cp_count = 0; 977*0Sstevel@tonic-gate cpi->cp_go = 0; 978*0Sstevel@tonic-gate for (i = 0; i < NCPU; i++) 979*0Sstevel@tonic-gate safe_list[i] = PAUSE_IDLE; 980*0Sstevel@tonic-gate kpreempt_disable(); 981*0Sstevel@tonic-gate 982*0Sstevel@tonic-gate /* 983*0Sstevel@tonic-gate * If running on the cpu that is going offline, get off it. 984*0Sstevel@tonic-gate * This is so that it won't be necessary to rechoose a CPU 985*0Sstevel@tonic-gate * when done. 986*0Sstevel@tonic-gate */ 987*0Sstevel@tonic-gate if (CPU == off_cp) 988*0Sstevel@tonic-gate cpu_id = off_cp->cpu_next_part->cpu_id; 989*0Sstevel@tonic-gate else 990*0Sstevel@tonic-gate cpu_id = CPU->cpu_id; 991*0Sstevel@tonic-gate affinity_set(cpu_id); 992*0Sstevel@tonic-gate 993*0Sstevel@tonic-gate /* 994*0Sstevel@tonic-gate * Start the pause threads and record how many were started 995*0Sstevel@tonic-gate */ 996*0Sstevel@tonic-gate cpi->cp_count = cpu_pause_start(cpu_id); 997*0Sstevel@tonic-gate 998*0Sstevel@tonic-gate /* 999*0Sstevel@tonic-gate * Now wait for all CPUs to be running the pause thread. 1000*0Sstevel@tonic-gate */ 1001*0Sstevel@tonic-gate while (cpi->cp_count > 0) { 1002*0Sstevel@tonic-gate /* 1003*0Sstevel@tonic-gate * Spin reading the count without grabbing the disp 1004*0Sstevel@tonic-gate * lock to make sure we don't prevent the pause 1005*0Sstevel@tonic-gate * threads from getting the lock. 1006*0Sstevel@tonic-gate */ 1007*0Sstevel@tonic-gate while (sema_held(&cpi->cp_sem)) 1008*0Sstevel@tonic-gate ; 1009*0Sstevel@tonic-gate if (sema_tryp(&cpi->cp_sem)) 1010*0Sstevel@tonic-gate --cpi->cp_count; 1011*0Sstevel@tonic-gate } 1012*0Sstevel@tonic-gate cpi->cp_go = 1; /* all have reached cpu_pause */ 1013*0Sstevel@tonic-gate 1014*0Sstevel@tonic-gate /* 1015*0Sstevel@tonic-gate * Now wait for all CPUs to spl. (Transition from PAUSE_READY 1016*0Sstevel@tonic-gate * to PAUSE_WAIT.) 1017*0Sstevel@tonic-gate */ 1018*0Sstevel@tonic-gate for (i = 0; i < NCPU; i++) { 1019*0Sstevel@tonic-gate while (safe_list[i] != PAUSE_WAIT) 1020*0Sstevel@tonic-gate ; 1021*0Sstevel@tonic-gate } 1022*0Sstevel@tonic-gate cpi->cp_spl = splhigh(); /* block dispatcher on this CPU */ 1023*0Sstevel@tonic-gate cpi->cp_paused = curthread; 1024*0Sstevel@tonic-gate } 1025*0Sstevel@tonic-gate 1026*0Sstevel@tonic-gate /* 1027*0Sstevel@tonic-gate * Check whether the current thread has CPUs paused 1028*0Sstevel@tonic-gate */ 1029*0Sstevel@tonic-gate int 1030*0Sstevel@tonic-gate cpus_paused(void) 1031*0Sstevel@tonic-gate { 1032*0Sstevel@tonic-gate if (cpu_pause_info.cp_paused != NULL) { 1033*0Sstevel@tonic-gate ASSERT(cpu_pause_info.cp_paused == curthread); 1034*0Sstevel@tonic-gate return (1); 1035*0Sstevel@tonic-gate } 1036*0Sstevel@tonic-gate return (0); 1037*0Sstevel@tonic-gate } 1038*0Sstevel@tonic-gate 1039*0Sstevel@tonic-gate static cpu_t * 1040*0Sstevel@tonic-gate cpu_get_all(processorid_t cpun) 1041*0Sstevel@tonic-gate { 1042*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1043*0Sstevel@tonic-gate 1044*0Sstevel@tonic-gate if (cpun >= NCPU || cpun < 0 || !CPU_IN_SET(cpu_available, cpun)) 1045*0Sstevel@tonic-gate return (NULL); 1046*0Sstevel@tonic-gate return (cpu[cpun]); 1047*0Sstevel@tonic-gate } 1048*0Sstevel@tonic-gate 1049*0Sstevel@tonic-gate /* 1050*0Sstevel@tonic-gate * Check whether cpun is a valid processor id and whether it should be 1051*0Sstevel@tonic-gate * visible from the current zone. If it is, return a pointer to the 1052*0Sstevel@tonic-gate * associated CPU structure. 1053*0Sstevel@tonic-gate */ 1054*0Sstevel@tonic-gate cpu_t * 1055*0Sstevel@tonic-gate cpu_get(processorid_t cpun) 1056*0Sstevel@tonic-gate { 1057*0Sstevel@tonic-gate cpu_t *c; 1058*0Sstevel@tonic-gate 1059*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1060*0Sstevel@tonic-gate c = cpu_get_all(cpun); 1061*0Sstevel@tonic-gate if (c != NULL && !INGLOBALZONE(curproc) && pool_pset_enabled() && 1062*0Sstevel@tonic-gate zone_pset_get(curproc->p_zone) != cpupart_query_cpu(c)) 1063*0Sstevel@tonic-gate return (NULL); 1064*0Sstevel@tonic-gate return (c); 1065*0Sstevel@tonic-gate } 1066*0Sstevel@tonic-gate 1067*0Sstevel@tonic-gate /* 1068*0Sstevel@tonic-gate * The following functions should be used to check CPU states in the kernel. 1069*0Sstevel@tonic-gate * They should be invoked with cpu_lock held. Kernel subsystems interested 1070*0Sstevel@tonic-gate * in CPU states should *not* use cpu_get_state() and various P_ONLINE/etc 1071*0Sstevel@tonic-gate * states. Those are for user-land (and system call) use only. 1072*0Sstevel@tonic-gate */ 1073*0Sstevel@tonic-gate 1074*0Sstevel@tonic-gate /* 1075*0Sstevel@tonic-gate * Determine whether the CPU is online and handling interrupts. 1076*0Sstevel@tonic-gate */ 1077*0Sstevel@tonic-gate int 1078*0Sstevel@tonic-gate cpu_is_online(cpu_t *cpu) 1079*0Sstevel@tonic-gate { 1080*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1081*0Sstevel@tonic-gate return (cpu_flagged_online(cpu->cpu_flags)); 1082*0Sstevel@tonic-gate } 1083*0Sstevel@tonic-gate 1084*0Sstevel@tonic-gate /* 1085*0Sstevel@tonic-gate * Determine whether the CPU is offline (this includes spare and faulted). 1086*0Sstevel@tonic-gate */ 1087*0Sstevel@tonic-gate int 1088*0Sstevel@tonic-gate cpu_is_offline(cpu_t *cpu) 1089*0Sstevel@tonic-gate { 1090*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1091*0Sstevel@tonic-gate return (cpu_flagged_offline(cpu->cpu_flags)); 1092*0Sstevel@tonic-gate } 1093*0Sstevel@tonic-gate 1094*0Sstevel@tonic-gate /* 1095*0Sstevel@tonic-gate * Determine whether the CPU is powered off. 1096*0Sstevel@tonic-gate */ 1097*0Sstevel@tonic-gate int 1098*0Sstevel@tonic-gate cpu_is_poweredoff(cpu_t *cpu) 1099*0Sstevel@tonic-gate { 1100*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1101*0Sstevel@tonic-gate return (cpu_flagged_poweredoff(cpu->cpu_flags)); 1102*0Sstevel@tonic-gate } 1103*0Sstevel@tonic-gate 1104*0Sstevel@tonic-gate /* 1105*0Sstevel@tonic-gate * Determine whether the CPU is handling interrupts. 1106*0Sstevel@tonic-gate */ 1107*0Sstevel@tonic-gate int 1108*0Sstevel@tonic-gate cpu_is_nointr(cpu_t *cpu) 1109*0Sstevel@tonic-gate { 1110*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1111*0Sstevel@tonic-gate return (cpu_flagged_nointr(cpu->cpu_flags)); 1112*0Sstevel@tonic-gate } 1113*0Sstevel@tonic-gate 1114*0Sstevel@tonic-gate /* 1115*0Sstevel@tonic-gate * Determine whether the CPU is active (scheduling threads). 1116*0Sstevel@tonic-gate */ 1117*0Sstevel@tonic-gate int 1118*0Sstevel@tonic-gate cpu_is_active(cpu_t *cpu) 1119*0Sstevel@tonic-gate { 1120*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1121*0Sstevel@tonic-gate return (cpu_flagged_active(cpu->cpu_flags)); 1122*0Sstevel@tonic-gate } 1123*0Sstevel@tonic-gate 1124*0Sstevel@tonic-gate /* 1125*0Sstevel@tonic-gate * Same as above, but these require cpu_flags instead of cpu_t pointers. 1126*0Sstevel@tonic-gate */ 1127*0Sstevel@tonic-gate int 1128*0Sstevel@tonic-gate cpu_flagged_online(cpu_flag_t cpu_flags) 1129*0Sstevel@tonic-gate { 1130*0Sstevel@tonic-gate return (cpu_flagged_active(cpu_flags) && 1131*0Sstevel@tonic-gate (cpu_flags & CPU_ENABLE)); 1132*0Sstevel@tonic-gate } 1133*0Sstevel@tonic-gate 1134*0Sstevel@tonic-gate int 1135*0Sstevel@tonic-gate cpu_flagged_offline(cpu_flag_t cpu_flags) 1136*0Sstevel@tonic-gate { 1137*0Sstevel@tonic-gate return (((cpu_flags & CPU_POWEROFF) == 0) && 1138*0Sstevel@tonic-gate ((cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY)); 1139*0Sstevel@tonic-gate } 1140*0Sstevel@tonic-gate 1141*0Sstevel@tonic-gate int 1142*0Sstevel@tonic-gate cpu_flagged_poweredoff(cpu_flag_t cpu_flags) 1143*0Sstevel@tonic-gate { 1144*0Sstevel@tonic-gate return ((cpu_flags & CPU_POWEROFF) == CPU_POWEROFF); 1145*0Sstevel@tonic-gate } 1146*0Sstevel@tonic-gate 1147*0Sstevel@tonic-gate int 1148*0Sstevel@tonic-gate cpu_flagged_nointr(cpu_flag_t cpu_flags) 1149*0Sstevel@tonic-gate { 1150*0Sstevel@tonic-gate return (cpu_flagged_active(cpu_flags) && 1151*0Sstevel@tonic-gate (cpu_flags & CPU_ENABLE) == 0); 1152*0Sstevel@tonic-gate } 1153*0Sstevel@tonic-gate 1154*0Sstevel@tonic-gate int 1155*0Sstevel@tonic-gate cpu_flagged_active(cpu_flag_t cpu_flags) 1156*0Sstevel@tonic-gate { 1157*0Sstevel@tonic-gate return (((cpu_flags & (CPU_POWEROFF | CPU_FAULTED | CPU_SPARE)) == 0) && 1158*0Sstevel@tonic-gate ((cpu_flags & (CPU_READY | CPU_OFFLINE)) == CPU_READY)); 1159*0Sstevel@tonic-gate } 1160*0Sstevel@tonic-gate 1161*0Sstevel@tonic-gate /* 1162*0Sstevel@tonic-gate * Bring the indicated CPU online. 1163*0Sstevel@tonic-gate */ 1164*0Sstevel@tonic-gate int 1165*0Sstevel@tonic-gate cpu_online(cpu_t *cp) 1166*0Sstevel@tonic-gate { 1167*0Sstevel@tonic-gate int error = 0; 1168*0Sstevel@tonic-gate 1169*0Sstevel@tonic-gate /* 1170*0Sstevel@tonic-gate * Handle on-line request. 1171*0Sstevel@tonic-gate * This code must put the new CPU on the active list before 1172*0Sstevel@tonic-gate * starting it because it will not be paused, and will start 1173*0Sstevel@tonic-gate * using the active list immediately. The real start occurs 1174*0Sstevel@tonic-gate * when the CPU_QUIESCED flag is turned off. 1175*0Sstevel@tonic-gate */ 1176*0Sstevel@tonic-gate 1177*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1178*0Sstevel@tonic-gate 1179*0Sstevel@tonic-gate /* 1180*0Sstevel@tonic-gate * Put all the cpus into a known safe place. 1181*0Sstevel@tonic-gate * No mutexes can be entered while CPUs are paused. 1182*0Sstevel@tonic-gate */ 1183*0Sstevel@tonic-gate error = mp_cpu_start(cp); /* arch-dep hook */ 1184*0Sstevel@tonic-gate if (error == 0) { 1185*0Sstevel@tonic-gate pause_cpus(NULL); 1186*0Sstevel@tonic-gate cpu_add_active_internal(cp); 1187*0Sstevel@tonic-gate if (cp->cpu_flags & CPU_FAULTED) { 1188*0Sstevel@tonic-gate cp->cpu_flags &= ~CPU_FAULTED; 1189*0Sstevel@tonic-gate mp_cpu_faulted_exit(cp); 1190*0Sstevel@tonic-gate } 1191*0Sstevel@tonic-gate cp->cpu_flags &= ~(CPU_QUIESCED | CPU_OFFLINE | CPU_FROZEN | 1192*0Sstevel@tonic-gate CPU_SPARE); 1193*0Sstevel@tonic-gate start_cpus(); 1194*0Sstevel@tonic-gate cpu_stats_kstat_create(cp); 1195*0Sstevel@tonic-gate cpu_create_intrstat(cp); 1196*0Sstevel@tonic-gate lgrp_kstat_create(cp); 1197*0Sstevel@tonic-gate cpu_state_change_notify(cp->cpu_id, CPU_ON); 1198*0Sstevel@tonic-gate cpu_intr_enable(cp); /* arch-dep hook */ 1199*0Sstevel@tonic-gate cyclic_online(cp); 1200*0Sstevel@tonic-gate poke_cpu(cp->cpu_id); 1201*0Sstevel@tonic-gate } 1202*0Sstevel@tonic-gate 1203*0Sstevel@tonic-gate return (error); 1204*0Sstevel@tonic-gate } 1205*0Sstevel@tonic-gate 1206*0Sstevel@tonic-gate /* 1207*0Sstevel@tonic-gate * Take the indicated CPU offline. 1208*0Sstevel@tonic-gate */ 1209*0Sstevel@tonic-gate int 1210*0Sstevel@tonic-gate cpu_offline(cpu_t *cp, int flags) 1211*0Sstevel@tonic-gate { 1212*0Sstevel@tonic-gate cpupart_t *pp; 1213*0Sstevel@tonic-gate int error = 0; 1214*0Sstevel@tonic-gate cpu_t *ncp; 1215*0Sstevel@tonic-gate int intr_enable; 1216*0Sstevel@tonic-gate int cyclic_off = 0; 1217*0Sstevel@tonic-gate int loop_count; 1218*0Sstevel@tonic-gate int no_quiesce = 0; 1219*0Sstevel@tonic-gate int (*bound_func)(struct cpu *, int); 1220*0Sstevel@tonic-gate kthread_t *t; 1221*0Sstevel@tonic-gate lpl_t *cpu_lpl; 1222*0Sstevel@tonic-gate proc_t *p; 1223*0Sstevel@tonic-gate int lgrp_diff_lpl; 1224*0Sstevel@tonic-gate 1225*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1226*0Sstevel@tonic-gate 1227*0Sstevel@tonic-gate /* 1228*0Sstevel@tonic-gate * If we're going from faulted or spare to offline, just 1229*0Sstevel@tonic-gate * clear these flags and update CPU state. 1230*0Sstevel@tonic-gate */ 1231*0Sstevel@tonic-gate if (cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) { 1232*0Sstevel@tonic-gate if (cp->cpu_flags & CPU_FAULTED) { 1233*0Sstevel@tonic-gate cp->cpu_flags &= ~CPU_FAULTED; 1234*0Sstevel@tonic-gate mp_cpu_faulted_exit(cp); 1235*0Sstevel@tonic-gate } 1236*0Sstevel@tonic-gate cp->cpu_flags &= ~CPU_SPARE; 1237*0Sstevel@tonic-gate cpu_set_state(cp); 1238*0Sstevel@tonic-gate return (0); 1239*0Sstevel@tonic-gate } 1240*0Sstevel@tonic-gate 1241*0Sstevel@tonic-gate /* 1242*0Sstevel@tonic-gate * Handle off-line request. 1243*0Sstevel@tonic-gate */ 1244*0Sstevel@tonic-gate pp = cp->cpu_part; 1245*0Sstevel@tonic-gate /* 1246*0Sstevel@tonic-gate * Don't offline last online CPU in partition 1247*0Sstevel@tonic-gate */ 1248*0Sstevel@tonic-gate if (ncpus_online <= 1 || pp->cp_ncpus <= 1 || cpu_intr_count(cp) < 2) 1249*0Sstevel@tonic-gate return (EBUSY); 1250*0Sstevel@tonic-gate /* 1251*0Sstevel@tonic-gate * Unbind all thread bound to our CPU if we were asked to. 1252*0Sstevel@tonic-gate */ 1253*0Sstevel@tonic-gate if (flags & CPU_FORCED && (error = cpu_unbind(cp->cpu_id)) != 0) 1254*0Sstevel@tonic-gate return (error); 1255*0Sstevel@tonic-gate /* 1256*0Sstevel@tonic-gate * We shouldn't be bound to this CPU ourselves. 1257*0Sstevel@tonic-gate */ 1258*0Sstevel@tonic-gate if (curthread->t_bound_cpu == cp) 1259*0Sstevel@tonic-gate return (EBUSY); 1260*0Sstevel@tonic-gate 1261*0Sstevel@tonic-gate /* 1262*0Sstevel@tonic-gate * Tell interested parties that this CPU is going offline. 1263*0Sstevel@tonic-gate */ 1264*0Sstevel@tonic-gate cpu_state_change_notify(cp->cpu_id, CPU_OFF); 1265*0Sstevel@tonic-gate 1266*0Sstevel@tonic-gate /* 1267*0Sstevel@tonic-gate * Take the CPU out of interrupt participation so we won't find 1268*0Sstevel@tonic-gate * bound kernel threads. If the architecture cannot completely 1269*0Sstevel@tonic-gate * shut off interrupts on the CPU, don't quiesce it, but don't 1270*0Sstevel@tonic-gate * run anything but interrupt thread... this is indicated by 1271*0Sstevel@tonic-gate * the CPU_OFFLINE flag being on but the CPU_QUIESCE flag being 1272*0Sstevel@tonic-gate * off. 1273*0Sstevel@tonic-gate */ 1274*0Sstevel@tonic-gate intr_enable = cp->cpu_flags & CPU_ENABLE; 1275*0Sstevel@tonic-gate if (intr_enable) 1276*0Sstevel@tonic-gate no_quiesce = cpu_intr_disable(cp); 1277*0Sstevel@tonic-gate 1278*0Sstevel@tonic-gate /* 1279*0Sstevel@tonic-gate * Record that we are aiming to offline this cpu. This acts as 1280*0Sstevel@tonic-gate * a barrier to further weak binding requests in thread_nomigrate 1281*0Sstevel@tonic-gate * and also causes cpu_choose, disp_lowpri_cpu and setfrontdq to 1282*0Sstevel@tonic-gate * lean away from this cpu. Further strong bindings are already 1283*0Sstevel@tonic-gate * avoided since we hold cpu_lock. Since threads that are set 1284*0Sstevel@tonic-gate * runnable around now and others coming off the target cpu are 1285*0Sstevel@tonic-gate * directed away from the target, existing strong and weak bindings 1286*0Sstevel@tonic-gate * (especially the latter) to the target cpu stand maximum chance of 1287*0Sstevel@tonic-gate * being able to unbind during the short delay loop below (if other 1288*0Sstevel@tonic-gate * unbound threads compete they may not see cpu in time to unbind 1289*0Sstevel@tonic-gate * even if they would do so immediately. 1290*0Sstevel@tonic-gate */ 1291*0Sstevel@tonic-gate cpu_inmotion = cp; 1292*0Sstevel@tonic-gate membar_enter(); 1293*0Sstevel@tonic-gate 1294*0Sstevel@tonic-gate /* 1295*0Sstevel@tonic-gate * Check for kernel threads (strong or weak) bound to that CPU. 1296*0Sstevel@tonic-gate * Strongly bound threads may not unbind, and we'll have to return 1297*0Sstevel@tonic-gate * EBUSY. Weakly bound threads should always disappear - we've 1298*0Sstevel@tonic-gate * stopped more weak binding with cpu_inmotion and existing 1299*0Sstevel@tonic-gate * bindings will drain imminently (they may not block). Nonetheless 1300*0Sstevel@tonic-gate * we will wait for a fixed period for all bound threads to disappear. 1301*0Sstevel@tonic-gate * Inactive interrupt threads are OK (they'll be in TS_FREE 1302*0Sstevel@tonic-gate * state). If test finds some bound threads, wait a few ticks 1303*0Sstevel@tonic-gate * to give short-lived threads (such as interrupts) chance to 1304*0Sstevel@tonic-gate * complete. Note that if no_quiesce is set, i.e. this cpu 1305*0Sstevel@tonic-gate * is required to service interrupts, then we take the route 1306*0Sstevel@tonic-gate * that permits interrupt threads to be active (or bypassed). 1307*0Sstevel@tonic-gate */ 1308*0Sstevel@tonic-gate bound_func = no_quiesce ? disp_bound_threads : disp_bound_anythreads; 1309*0Sstevel@tonic-gate 1310*0Sstevel@tonic-gate again: for (loop_count = 0; (*bound_func)(cp, 0); loop_count++) { 1311*0Sstevel@tonic-gate if (loop_count >= 5) { 1312*0Sstevel@tonic-gate error = EBUSY; /* some threads still bound */ 1313*0Sstevel@tonic-gate break; 1314*0Sstevel@tonic-gate } 1315*0Sstevel@tonic-gate 1316*0Sstevel@tonic-gate /* 1317*0Sstevel@tonic-gate * If some threads were assigned, give them 1318*0Sstevel@tonic-gate * a chance to complete or move. 1319*0Sstevel@tonic-gate * 1320*0Sstevel@tonic-gate * This assumes that the clock_thread is not bound 1321*0Sstevel@tonic-gate * to any CPU, because the clock_thread is needed to 1322*0Sstevel@tonic-gate * do the delay(hz/100). 1323*0Sstevel@tonic-gate * 1324*0Sstevel@tonic-gate * Note: we still hold the cpu_lock while waiting for 1325*0Sstevel@tonic-gate * the next clock tick. This is OK since it isn't 1326*0Sstevel@tonic-gate * needed for anything else except processor_bind(2), 1327*0Sstevel@tonic-gate * and system initialization. If we drop the lock, 1328*0Sstevel@tonic-gate * we would risk another p_online disabling the last 1329*0Sstevel@tonic-gate * processor. 1330*0Sstevel@tonic-gate */ 1331*0Sstevel@tonic-gate delay(hz/100); 1332*0Sstevel@tonic-gate } 1333*0Sstevel@tonic-gate 1334*0Sstevel@tonic-gate if (error == 0 && cyclic_off == 0) { 1335*0Sstevel@tonic-gate if (!cyclic_offline(cp)) { 1336*0Sstevel@tonic-gate /* 1337*0Sstevel@tonic-gate * We must have bound cyclics... 1338*0Sstevel@tonic-gate */ 1339*0Sstevel@tonic-gate error = EBUSY; 1340*0Sstevel@tonic-gate goto out; 1341*0Sstevel@tonic-gate } 1342*0Sstevel@tonic-gate cyclic_off = 1; 1343*0Sstevel@tonic-gate } 1344*0Sstevel@tonic-gate 1345*0Sstevel@tonic-gate /* 1346*0Sstevel@tonic-gate * Call mp_cpu_stop() to perform any special operations 1347*0Sstevel@tonic-gate * needed for this machine architecture to offline a CPU. 1348*0Sstevel@tonic-gate */ 1349*0Sstevel@tonic-gate if (error == 0) 1350*0Sstevel@tonic-gate error = mp_cpu_stop(cp); /* arch-dep hook */ 1351*0Sstevel@tonic-gate 1352*0Sstevel@tonic-gate /* 1353*0Sstevel@tonic-gate * If that all worked, take the CPU offline and decrement 1354*0Sstevel@tonic-gate * ncpus_online. 1355*0Sstevel@tonic-gate */ 1356*0Sstevel@tonic-gate if (error == 0) { 1357*0Sstevel@tonic-gate /* 1358*0Sstevel@tonic-gate * Put all the cpus into a known safe place. 1359*0Sstevel@tonic-gate * No mutexes can be entered while CPUs are paused. 1360*0Sstevel@tonic-gate */ 1361*0Sstevel@tonic-gate pause_cpus(cp); 1362*0Sstevel@tonic-gate /* 1363*0Sstevel@tonic-gate * Repeat the operation, if necessary, to make sure that 1364*0Sstevel@tonic-gate * all outstanding low-level interrupts run to completion 1365*0Sstevel@tonic-gate * before we set the CPU_QUIESCED flag. It's also possible 1366*0Sstevel@tonic-gate * that a thread has weak bound to the cpu despite our raising 1367*0Sstevel@tonic-gate * cpu_inmotion above since it may have loaded that 1368*0Sstevel@tonic-gate * value before the barrier became visible (this would have 1369*0Sstevel@tonic-gate * to be the thread that was on the target cpu at the time 1370*0Sstevel@tonic-gate * we raised the barrier). 1371*0Sstevel@tonic-gate */ 1372*0Sstevel@tonic-gate if ((!no_quiesce && cp->cpu_intr_actv != 0) || 1373*0Sstevel@tonic-gate (*bound_func)(cp, 1)) { 1374*0Sstevel@tonic-gate start_cpus(); 1375*0Sstevel@tonic-gate (void) mp_cpu_start(cp); 1376*0Sstevel@tonic-gate goto again; 1377*0Sstevel@tonic-gate } 1378*0Sstevel@tonic-gate ncp = cp->cpu_next_part; 1379*0Sstevel@tonic-gate cpu_lpl = cp->cpu_lpl; 1380*0Sstevel@tonic-gate ASSERT(cpu_lpl != NULL); 1381*0Sstevel@tonic-gate 1382*0Sstevel@tonic-gate /* 1383*0Sstevel@tonic-gate * Remove the CPU from the list of active CPUs. 1384*0Sstevel@tonic-gate */ 1385*0Sstevel@tonic-gate cpu_remove_active(cp); 1386*0Sstevel@tonic-gate 1387*0Sstevel@tonic-gate /* 1388*0Sstevel@tonic-gate * Walk the active process list and look for threads 1389*0Sstevel@tonic-gate * whose home lgroup needs to be updated, or 1390*0Sstevel@tonic-gate * the last CPU they run on is the one being offlined now. 1391*0Sstevel@tonic-gate */ 1392*0Sstevel@tonic-gate 1393*0Sstevel@tonic-gate ASSERT(curthread->t_cpu != cp); 1394*0Sstevel@tonic-gate for (p = practive; p != NULL; p = p->p_next) { 1395*0Sstevel@tonic-gate 1396*0Sstevel@tonic-gate t = p->p_tlist; 1397*0Sstevel@tonic-gate 1398*0Sstevel@tonic-gate if (t == NULL) 1399*0Sstevel@tonic-gate continue; 1400*0Sstevel@tonic-gate 1401*0Sstevel@tonic-gate lgrp_diff_lpl = 0; 1402*0Sstevel@tonic-gate 1403*0Sstevel@tonic-gate do { 1404*0Sstevel@tonic-gate ASSERT(t->t_lpl != NULL); 1405*0Sstevel@tonic-gate /* 1406*0Sstevel@tonic-gate * Taking last CPU in lpl offline 1407*0Sstevel@tonic-gate * Rehome thread if it is in this lpl 1408*0Sstevel@tonic-gate * Otherwise, update the count of how many 1409*0Sstevel@tonic-gate * threads are in this CPU's lgroup but have 1410*0Sstevel@tonic-gate * a different lpl. 1411*0Sstevel@tonic-gate */ 1412*0Sstevel@tonic-gate 1413*0Sstevel@tonic-gate if (cpu_lpl->lpl_ncpu == 0) { 1414*0Sstevel@tonic-gate if (t->t_lpl == cpu_lpl) 1415*0Sstevel@tonic-gate lgrp_move_thread(t, 1416*0Sstevel@tonic-gate lgrp_choose(t, 1417*0Sstevel@tonic-gate t->t_cpupart), 0); 1418*0Sstevel@tonic-gate else if (t->t_lpl->lpl_lgrpid == 1419*0Sstevel@tonic-gate cpu_lpl->lpl_lgrpid) 1420*0Sstevel@tonic-gate lgrp_diff_lpl++; 1421*0Sstevel@tonic-gate } 1422*0Sstevel@tonic-gate ASSERT(t->t_lpl->lpl_ncpu > 0); 1423*0Sstevel@tonic-gate 1424*0Sstevel@tonic-gate /* 1425*0Sstevel@tonic-gate * Update CPU last ran on if it was this CPU 1426*0Sstevel@tonic-gate */ 1427*0Sstevel@tonic-gate if (t->t_cpu == cp && t->t_bound_cpu != cp) 1428*0Sstevel@tonic-gate t->t_cpu = disp_lowpri_cpu(ncp, t->t_lpl, 1429*0Sstevel@tonic-gate t->t_pri, NULL); 1430*0Sstevel@tonic-gate ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp || 1431*0Sstevel@tonic-gate t->t_weakbound_cpu == cp); 1432*0Sstevel@tonic-gate 1433*0Sstevel@tonic-gate t = t->t_forw; 1434*0Sstevel@tonic-gate } while (t != p->p_tlist); 1435*0Sstevel@tonic-gate 1436*0Sstevel@tonic-gate /* 1437*0Sstevel@tonic-gate * Didn't find any threads in the same lgroup as this 1438*0Sstevel@tonic-gate * CPU with a different lpl, so remove the lgroup from 1439*0Sstevel@tonic-gate * the process lgroup bitmask. 1440*0Sstevel@tonic-gate */ 1441*0Sstevel@tonic-gate 1442*0Sstevel@tonic-gate if (lgrp_diff_lpl == 0) 1443*0Sstevel@tonic-gate klgrpset_del(p->p_lgrpset, cpu_lpl->lpl_lgrpid); 1444*0Sstevel@tonic-gate } 1445*0Sstevel@tonic-gate 1446*0Sstevel@tonic-gate /* 1447*0Sstevel@tonic-gate * Walk thread list looking for threads that need to be 1448*0Sstevel@tonic-gate * rehomed, since there are some threads that are not in 1449*0Sstevel@tonic-gate * their process's p_tlist. 1450*0Sstevel@tonic-gate */ 1451*0Sstevel@tonic-gate 1452*0Sstevel@tonic-gate t = curthread; 1453*0Sstevel@tonic-gate do { 1454*0Sstevel@tonic-gate ASSERT(t != NULL && t->t_lpl != NULL); 1455*0Sstevel@tonic-gate 1456*0Sstevel@tonic-gate /* 1457*0Sstevel@tonic-gate * Rehome threads with same lpl as this CPU when this 1458*0Sstevel@tonic-gate * is the last CPU in the lpl. 1459*0Sstevel@tonic-gate */ 1460*0Sstevel@tonic-gate 1461*0Sstevel@tonic-gate if ((cpu_lpl->lpl_ncpu == 0) && (t->t_lpl == cpu_lpl)) 1462*0Sstevel@tonic-gate lgrp_move_thread(t, 1463*0Sstevel@tonic-gate lgrp_choose(t, t->t_cpupart), 1); 1464*0Sstevel@tonic-gate 1465*0Sstevel@tonic-gate ASSERT(t->t_lpl->lpl_ncpu > 0); 1466*0Sstevel@tonic-gate 1467*0Sstevel@tonic-gate /* 1468*0Sstevel@tonic-gate * Update CPU last ran on if it was this CPU 1469*0Sstevel@tonic-gate */ 1470*0Sstevel@tonic-gate 1471*0Sstevel@tonic-gate if (t->t_cpu == cp && t->t_bound_cpu != cp) { 1472*0Sstevel@tonic-gate t->t_cpu = disp_lowpri_cpu(ncp, 1473*0Sstevel@tonic-gate t->t_lpl, t->t_pri, NULL); 1474*0Sstevel@tonic-gate } 1475*0Sstevel@tonic-gate ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp || 1476*0Sstevel@tonic-gate t->t_weakbound_cpu == cp); 1477*0Sstevel@tonic-gate t = t->t_next; 1478*0Sstevel@tonic-gate 1479*0Sstevel@tonic-gate } while (t != curthread); 1480*0Sstevel@tonic-gate ASSERT((cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) == 0); 1481*0Sstevel@tonic-gate cp->cpu_flags |= CPU_OFFLINE; 1482*0Sstevel@tonic-gate disp_cpu_inactive(cp); 1483*0Sstevel@tonic-gate if (!no_quiesce) 1484*0Sstevel@tonic-gate cp->cpu_flags |= CPU_QUIESCED; 1485*0Sstevel@tonic-gate ncpus_online--; 1486*0Sstevel@tonic-gate cpu_set_state(cp); 1487*0Sstevel@tonic-gate cpu_inmotion = NULL; 1488*0Sstevel@tonic-gate start_cpus(); 1489*0Sstevel@tonic-gate cpu_stats_kstat_destroy(cp); 1490*0Sstevel@tonic-gate cpu_delete_intrstat(cp); 1491*0Sstevel@tonic-gate lgrp_kstat_destroy(cp); 1492*0Sstevel@tonic-gate } 1493*0Sstevel@tonic-gate 1494*0Sstevel@tonic-gate out: 1495*0Sstevel@tonic-gate cpu_inmotion = NULL; 1496*0Sstevel@tonic-gate 1497*0Sstevel@tonic-gate /* 1498*0Sstevel@tonic-gate * If we failed, re-enable interrupts. 1499*0Sstevel@tonic-gate * Do this even if cpu_intr_disable returned an error, because 1500*0Sstevel@tonic-gate * it may have partially disabled interrupts. 1501*0Sstevel@tonic-gate */ 1502*0Sstevel@tonic-gate if (error && intr_enable) 1503*0Sstevel@tonic-gate cpu_intr_enable(cp); 1504*0Sstevel@tonic-gate 1505*0Sstevel@tonic-gate /* 1506*0Sstevel@tonic-gate * If we failed, but managed to offline the cyclic subsystem on this 1507*0Sstevel@tonic-gate * CPU, bring it back online. 1508*0Sstevel@tonic-gate */ 1509*0Sstevel@tonic-gate if (error && cyclic_off) 1510*0Sstevel@tonic-gate cyclic_online(cp); 1511*0Sstevel@tonic-gate 1512*0Sstevel@tonic-gate /* 1513*0Sstevel@tonic-gate * If we failed, we need to notify everyone that this CPU is back on. 1514*0Sstevel@tonic-gate */ 1515*0Sstevel@tonic-gate if (error != 0) 1516*0Sstevel@tonic-gate cpu_state_change_notify(cp->cpu_id, CPU_ON); 1517*0Sstevel@tonic-gate 1518*0Sstevel@tonic-gate return (error); 1519*0Sstevel@tonic-gate } 1520*0Sstevel@tonic-gate 1521*0Sstevel@tonic-gate /* 1522*0Sstevel@tonic-gate * Mark the indicated CPU as faulted, taking it offline. 1523*0Sstevel@tonic-gate */ 1524*0Sstevel@tonic-gate int 1525*0Sstevel@tonic-gate cpu_faulted(cpu_t *cp, int flags) 1526*0Sstevel@tonic-gate { 1527*0Sstevel@tonic-gate int error = 0; 1528*0Sstevel@tonic-gate 1529*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1530*0Sstevel@tonic-gate ASSERT(!cpu_is_poweredoff(cp)); 1531*0Sstevel@tonic-gate 1532*0Sstevel@tonic-gate if (cpu_is_offline(cp)) { 1533*0Sstevel@tonic-gate cp->cpu_flags &= ~CPU_SPARE; 1534*0Sstevel@tonic-gate cp->cpu_flags |= CPU_FAULTED; 1535*0Sstevel@tonic-gate mp_cpu_faulted_enter(cp); 1536*0Sstevel@tonic-gate cpu_set_state(cp); 1537*0Sstevel@tonic-gate return (0); 1538*0Sstevel@tonic-gate } 1539*0Sstevel@tonic-gate 1540*0Sstevel@tonic-gate if ((error = cpu_offline(cp, flags)) == 0) { 1541*0Sstevel@tonic-gate cp->cpu_flags |= CPU_FAULTED; 1542*0Sstevel@tonic-gate mp_cpu_faulted_enter(cp); 1543*0Sstevel@tonic-gate cpu_set_state(cp); 1544*0Sstevel@tonic-gate } 1545*0Sstevel@tonic-gate 1546*0Sstevel@tonic-gate return (error); 1547*0Sstevel@tonic-gate } 1548*0Sstevel@tonic-gate 1549*0Sstevel@tonic-gate /* 1550*0Sstevel@tonic-gate * Mark the indicated CPU as a spare, taking it offline. 1551*0Sstevel@tonic-gate */ 1552*0Sstevel@tonic-gate int 1553*0Sstevel@tonic-gate cpu_spare(cpu_t *cp, int flags) 1554*0Sstevel@tonic-gate { 1555*0Sstevel@tonic-gate int error = 0; 1556*0Sstevel@tonic-gate 1557*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1558*0Sstevel@tonic-gate ASSERT(!cpu_is_poweredoff(cp)); 1559*0Sstevel@tonic-gate 1560*0Sstevel@tonic-gate if (cpu_is_offline(cp)) { 1561*0Sstevel@tonic-gate if (cp->cpu_flags & CPU_FAULTED) { 1562*0Sstevel@tonic-gate cp->cpu_flags &= ~CPU_FAULTED; 1563*0Sstevel@tonic-gate mp_cpu_faulted_exit(cp); 1564*0Sstevel@tonic-gate } 1565*0Sstevel@tonic-gate cp->cpu_flags |= CPU_SPARE; 1566*0Sstevel@tonic-gate cpu_set_state(cp); 1567*0Sstevel@tonic-gate return (0); 1568*0Sstevel@tonic-gate } 1569*0Sstevel@tonic-gate 1570*0Sstevel@tonic-gate if ((error = cpu_offline(cp, flags)) == 0) { 1571*0Sstevel@tonic-gate cp->cpu_flags |= CPU_SPARE; 1572*0Sstevel@tonic-gate cpu_set_state(cp); 1573*0Sstevel@tonic-gate } 1574*0Sstevel@tonic-gate 1575*0Sstevel@tonic-gate return (error); 1576*0Sstevel@tonic-gate } 1577*0Sstevel@tonic-gate 1578*0Sstevel@tonic-gate /* 1579*0Sstevel@tonic-gate * Take the indicated CPU from poweroff to offline. 1580*0Sstevel@tonic-gate */ 1581*0Sstevel@tonic-gate int 1582*0Sstevel@tonic-gate cpu_poweron(cpu_t *cp) 1583*0Sstevel@tonic-gate { 1584*0Sstevel@tonic-gate int error = ENOTSUP; 1585*0Sstevel@tonic-gate 1586*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1587*0Sstevel@tonic-gate ASSERT(cpu_is_poweredoff(cp)); 1588*0Sstevel@tonic-gate 1589*0Sstevel@tonic-gate error = mp_cpu_poweron(cp); /* arch-dep hook */ 1590*0Sstevel@tonic-gate if (error == 0) 1591*0Sstevel@tonic-gate cpu_set_state(cp); 1592*0Sstevel@tonic-gate 1593*0Sstevel@tonic-gate return (error); 1594*0Sstevel@tonic-gate } 1595*0Sstevel@tonic-gate 1596*0Sstevel@tonic-gate /* 1597*0Sstevel@tonic-gate * Take the indicated CPU from any inactive state to powered off. 1598*0Sstevel@tonic-gate */ 1599*0Sstevel@tonic-gate int 1600*0Sstevel@tonic-gate cpu_poweroff(cpu_t *cp) 1601*0Sstevel@tonic-gate { 1602*0Sstevel@tonic-gate int error = ENOTSUP; 1603*0Sstevel@tonic-gate 1604*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1605*0Sstevel@tonic-gate ASSERT(cpu_is_offline(cp)); 1606*0Sstevel@tonic-gate 1607*0Sstevel@tonic-gate if (!(cp->cpu_flags & CPU_QUIESCED)) 1608*0Sstevel@tonic-gate return (EBUSY); /* not completely idle */ 1609*0Sstevel@tonic-gate 1610*0Sstevel@tonic-gate error = mp_cpu_poweroff(cp); /* arch-dep hook */ 1611*0Sstevel@tonic-gate if (error == 0) 1612*0Sstevel@tonic-gate cpu_set_state(cp); 1613*0Sstevel@tonic-gate 1614*0Sstevel@tonic-gate return (error); 1615*0Sstevel@tonic-gate } 1616*0Sstevel@tonic-gate 1617*0Sstevel@tonic-gate /* 1618*0Sstevel@tonic-gate * Initialize the CPU lists for the first CPU. 1619*0Sstevel@tonic-gate */ 1620*0Sstevel@tonic-gate void 1621*0Sstevel@tonic-gate cpu_list_init(cpu_t *cp) 1622*0Sstevel@tonic-gate { 1623*0Sstevel@tonic-gate cp->cpu_next = cp; 1624*0Sstevel@tonic-gate cp->cpu_prev = cp; 1625*0Sstevel@tonic-gate cpu_list = cp; 1626*0Sstevel@tonic-gate 1627*0Sstevel@tonic-gate cp->cpu_next_onln = cp; 1628*0Sstevel@tonic-gate cp->cpu_prev_onln = cp; 1629*0Sstevel@tonic-gate cpu_active = cp; 1630*0Sstevel@tonic-gate 1631*0Sstevel@tonic-gate cp->cpu_seqid = 0; 1632*0Sstevel@tonic-gate CPUSET_ADD(cpu_seqid_inuse, 0); 1633*0Sstevel@tonic-gate cp->cpu_cache_offset = KMEM_CACHE_SIZE(cp->cpu_seqid); 1634*0Sstevel@tonic-gate cp_default.cp_cpulist = cp; 1635*0Sstevel@tonic-gate cp_default.cp_ncpus = 1; 1636*0Sstevel@tonic-gate cp->cpu_next_lpl = cp; 1637*0Sstevel@tonic-gate cp->cpu_prev_lpl = cp; 1638*0Sstevel@tonic-gate cp->cpu_next_lgrp = cp; 1639*0Sstevel@tonic-gate cp->cpu_prev_lgrp = cp; 1640*0Sstevel@tonic-gate cp->cpu_next_part = cp; 1641*0Sstevel@tonic-gate cp->cpu_prev_part = cp; 1642*0Sstevel@tonic-gate cp->cpu_part = &cp_default; 1643*0Sstevel@tonic-gate cp->cpu_lpl = lpl_bootstrap; 1644*0Sstevel@tonic-gate 1645*0Sstevel@tonic-gate CPUSET_ADD(cpu_available, cp->cpu_id); 1646*0Sstevel@tonic-gate } 1647*0Sstevel@tonic-gate 1648*0Sstevel@tonic-gate /* 1649*0Sstevel@tonic-gate * Insert a CPU into the list of available CPUs. 1650*0Sstevel@tonic-gate */ 1651*0Sstevel@tonic-gate void 1652*0Sstevel@tonic-gate cpu_add_unit(cpu_t *cp) 1653*0Sstevel@tonic-gate { 1654*0Sstevel@tonic-gate int seqid; 1655*0Sstevel@tonic-gate 1656*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1657*0Sstevel@tonic-gate ASSERT(cpu_list != NULL); /* list started in cpu_list_init */ 1658*0Sstevel@tonic-gate 1659*0Sstevel@tonic-gate lgrp_config(LGRP_CONFIG_CPU_ADD, (uintptr_t)cp, 0); 1660*0Sstevel@tonic-gate 1661*0Sstevel@tonic-gate /* 1662*0Sstevel@tonic-gate * Note: most users of the cpu_list will grab the 1663*0Sstevel@tonic-gate * cpu_lock to insure that it isn't modified. However, 1664*0Sstevel@tonic-gate * certain users can't or won't do that. To allow this 1665*0Sstevel@tonic-gate * we pause the other cpus. Users who walk the list 1666*0Sstevel@tonic-gate * without cpu_lock, must disable kernel preemption 1667*0Sstevel@tonic-gate * to insure that the list isn't modified underneath 1668*0Sstevel@tonic-gate * them. Also, any cached pointers to cpu structures 1669*0Sstevel@tonic-gate * must be revalidated by checking to see if the 1670*0Sstevel@tonic-gate * cpu_next pointer points to itself. This check must 1671*0Sstevel@tonic-gate * be done with the cpu_lock held or kernel preemption 1672*0Sstevel@tonic-gate * disabled. This check relies upon the fact that 1673*0Sstevel@tonic-gate * old cpu structures are not free'ed or cleared after 1674*0Sstevel@tonic-gate * then are removed from the cpu_list. 1675*0Sstevel@tonic-gate * 1676*0Sstevel@tonic-gate * Note that the clock code walks the cpu list dereferencing 1677*0Sstevel@tonic-gate * the cpu_part pointer, so we need to initialize it before 1678*0Sstevel@tonic-gate * adding the cpu to the list. 1679*0Sstevel@tonic-gate */ 1680*0Sstevel@tonic-gate cp->cpu_part = &cp_default; 1681*0Sstevel@tonic-gate (void) pause_cpus(NULL); 1682*0Sstevel@tonic-gate cp->cpu_next = cpu_list; 1683*0Sstevel@tonic-gate cp->cpu_prev = cpu_list->cpu_prev; 1684*0Sstevel@tonic-gate cpu_list->cpu_prev->cpu_next = cp; 1685*0Sstevel@tonic-gate cpu_list->cpu_prev = cp; 1686*0Sstevel@tonic-gate start_cpus(); 1687*0Sstevel@tonic-gate 1688*0Sstevel@tonic-gate for (seqid = 0; CPU_IN_SET(cpu_seqid_inuse, seqid); seqid++) 1689*0Sstevel@tonic-gate continue; 1690*0Sstevel@tonic-gate CPUSET_ADD(cpu_seqid_inuse, seqid); 1691*0Sstevel@tonic-gate cp->cpu_seqid = seqid; 1692*0Sstevel@tonic-gate ASSERT(ncpus < max_ncpus); 1693*0Sstevel@tonic-gate ncpus++; 1694*0Sstevel@tonic-gate cp->cpu_cache_offset = KMEM_CACHE_SIZE(cp->cpu_seqid); 1695*0Sstevel@tonic-gate cpu[cp->cpu_id] = cp; 1696*0Sstevel@tonic-gate CPUSET_ADD(cpu_available, cp->cpu_id); 1697*0Sstevel@tonic-gate 1698*0Sstevel@tonic-gate /* 1699*0Sstevel@tonic-gate * allocate a pause thread for this CPU. 1700*0Sstevel@tonic-gate */ 1701*0Sstevel@tonic-gate cpu_pause_alloc(cp); 1702*0Sstevel@tonic-gate 1703*0Sstevel@tonic-gate /* 1704*0Sstevel@tonic-gate * So that new CPUs won't have NULL prev_onln and next_onln pointers, 1705*0Sstevel@tonic-gate * link them into a list of just that CPU. 1706*0Sstevel@tonic-gate * This is so that disp_lowpri_cpu will work for thread_create in 1707*0Sstevel@tonic-gate * pause_cpus() when called from the startup thread in a new CPU. 1708*0Sstevel@tonic-gate */ 1709*0Sstevel@tonic-gate cp->cpu_next_onln = cp; 1710*0Sstevel@tonic-gate cp->cpu_prev_onln = cp; 1711*0Sstevel@tonic-gate cpu_info_kstat_create(cp); 1712*0Sstevel@tonic-gate cp->cpu_next_part = cp; 1713*0Sstevel@tonic-gate cp->cpu_prev_part = cp; 1714*0Sstevel@tonic-gate 1715*0Sstevel@tonic-gate cp->cpu_next_lgrp = cp; 1716*0Sstevel@tonic-gate cp->cpu_prev_lgrp = cp; 1717*0Sstevel@tonic-gate cp->cpu_next_lpl = cp; 1718*0Sstevel@tonic-gate cp->cpu_prev_lpl = cp; 1719*0Sstevel@tonic-gate cp->cpu_lpl = lpl_bootstrap; 1720*0Sstevel@tonic-gate 1721*0Sstevel@tonic-gate init_cpu_mstate(cp, CMS_SYSTEM); 1722*0Sstevel@tonic-gate 1723*0Sstevel@tonic-gate pool_pset_mod = gethrtime(); 1724*0Sstevel@tonic-gate } 1725*0Sstevel@tonic-gate 1726*0Sstevel@tonic-gate /* 1727*0Sstevel@tonic-gate * Do the opposite of cpu_add_unit(). 1728*0Sstevel@tonic-gate */ 1729*0Sstevel@tonic-gate void 1730*0Sstevel@tonic-gate cpu_del_unit(int cpuid) 1731*0Sstevel@tonic-gate { 1732*0Sstevel@tonic-gate struct cpu *cp, *cpnext; 1733*0Sstevel@tonic-gate 1734*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1735*0Sstevel@tonic-gate cp = cpu[cpuid]; 1736*0Sstevel@tonic-gate ASSERT(cp != NULL); 1737*0Sstevel@tonic-gate 1738*0Sstevel@tonic-gate ASSERT(cp->cpu_next_onln == cp); 1739*0Sstevel@tonic-gate ASSERT(cp->cpu_prev_onln == cp); 1740*0Sstevel@tonic-gate ASSERT(cp->cpu_next_part == cp); 1741*0Sstevel@tonic-gate ASSERT(cp->cpu_prev_part == cp); 1742*0Sstevel@tonic-gate 1743*0Sstevel@tonic-gate chip_cpu_fini(cp); 1744*0Sstevel@tonic-gate 1745*0Sstevel@tonic-gate /* 1746*0Sstevel@tonic-gate * Destroy kstat stuff. 1747*0Sstevel@tonic-gate */ 1748*0Sstevel@tonic-gate cpu_info_kstat_destroy(cp); 1749*0Sstevel@tonic-gate term_cpu_mstate(cp); 1750*0Sstevel@tonic-gate /* 1751*0Sstevel@tonic-gate * Free up pause thread. 1752*0Sstevel@tonic-gate */ 1753*0Sstevel@tonic-gate cpu_pause_free(cp); 1754*0Sstevel@tonic-gate CPUSET_DEL(cpu_available, cp->cpu_id); 1755*0Sstevel@tonic-gate cpu[cp->cpu_id] = NULL; 1756*0Sstevel@tonic-gate /* 1757*0Sstevel@tonic-gate * The clock thread and mutex_vector_enter cannot hold the 1758*0Sstevel@tonic-gate * cpu_lock while traversing the cpu list, therefore we pause 1759*0Sstevel@tonic-gate * all other threads by pausing the other cpus. These, and any 1760*0Sstevel@tonic-gate * other routines holding cpu pointers while possibly sleeping 1761*0Sstevel@tonic-gate * must be sure to call kpreempt_disable before processing the 1762*0Sstevel@tonic-gate * list and be sure to check that the cpu has not been deleted 1763*0Sstevel@tonic-gate * after any sleeps (check cp->cpu_next != NULL). We guarantee 1764*0Sstevel@tonic-gate * to keep the deleted cpu structure around. 1765*0Sstevel@tonic-gate * 1766*0Sstevel@tonic-gate * Note that this MUST be done AFTER cpu_available 1767*0Sstevel@tonic-gate * has been updated so that we don't waste time 1768*0Sstevel@tonic-gate * trying to pause the cpu we're trying to delete. 1769*0Sstevel@tonic-gate */ 1770*0Sstevel@tonic-gate (void) pause_cpus(NULL); 1771*0Sstevel@tonic-gate 1772*0Sstevel@tonic-gate cpnext = cp->cpu_next; 1773*0Sstevel@tonic-gate cp->cpu_prev->cpu_next = cp->cpu_next; 1774*0Sstevel@tonic-gate cp->cpu_next->cpu_prev = cp->cpu_prev; 1775*0Sstevel@tonic-gate if (cp == cpu_list) 1776*0Sstevel@tonic-gate cpu_list = cpnext; 1777*0Sstevel@tonic-gate 1778*0Sstevel@tonic-gate /* 1779*0Sstevel@tonic-gate * Signals that the cpu has been deleted (see above). 1780*0Sstevel@tonic-gate */ 1781*0Sstevel@tonic-gate cp->cpu_next = NULL; 1782*0Sstevel@tonic-gate cp->cpu_prev = NULL; 1783*0Sstevel@tonic-gate 1784*0Sstevel@tonic-gate start_cpus(); 1785*0Sstevel@tonic-gate 1786*0Sstevel@tonic-gate CPUSET_DEL(cpu_seqid_inuse, cp->cpu_seqid); 1787*0Sstevel@tonic-gate ncpus--; 1788*0Sstevel@tonic-gate lgrp_config(LGRP_CONFIG_CPU_DEL, (uintptr_t)cp, 0); 1789*0Sstevel@tonic-gate 1790*0Sstevel@tonic-gate pool_pset_mod = gethrtime(); 1791*0Sstevel@tonic-gate } 1792*0Sstevel@tonic-gate 1793*0Sstevel@tonic-gate /* 1794*0Sstevel@tonic-gate * Add a CPU to the list of active CPUs. 1795*0Sstevel@tonic-gate * This routine must not get any locks, because other CPUs are paused. 1796*0Sstevel@tonic-gate */ 1797*0Sstevel@tonic-gate static void 1798*0Sstevel@tonic-gate cpu_add_active_internal(cpu_t *cp) 1799*0Sstevel@tonic-gate { 1800*0Sstevel@tonic-gate cpupart_t *pp = cp->cpu_part; 1801*0Sstevel@tonic-gate 1802*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1803*0Sstevel@tonic-gate ASSERT(cpu_list != NULL); /* list started in cpu_list_init */ 1804*0Sstevel@tonic-gate 1805*0Sstevel@tonic-gate ncpus_online++; 1806*0Sstevel@tonic-gate cpu_set_state(cp); 1807*0Sstevel@tonic-gate cp->cpu_next_onln = cpu_active; 1808*0Sstevel@tonic-gate cp->cpu_prev_onln = cpu_active->cpu_prev_onln; 1809*0Sstevel@tonic-gate cpu_active->cpu_prev_onln->cpu_next_onln = cp; 1810*0Sstevel@tonic-gate cpu_active->cpu_prev_onln = cp; 1811*0Sstevel@tonic-gate 1812*0Sstevel@tonic-gate if (pp->cp_cpulist) { 1813*0Sstevel@tonic-gate cp->cpu_next_part = pp->cp_cpulist; 1814*0Sstevel@tonic-gate cp->cpu_prev_part = pp->cp_cpulist->cpu_prev_part; 1815*0Sstevel@tonic-gate pp->cp_cpulist->cpu_prev_part->cpu_next_part = cp; 1816*0Sstevel@tonic-gate pp->cp_cpulist->cpu_prev_part = cp; 1817*0Sstevel@tonic-gate } else { 1818*0Sstevel@tonic-gate ASSERT(pp->cp_ncpus == 0); 1819*0Sstevel@tonic-gate pp->cp_cpulist = cp->cpu_next_part = cp->cpu_prev_part = cp; 1820*0Sstevel@tonic-gate } 1821*0Sstevel@tonic-gate pp->cp_ncpus++; 1822*0Sstevel@tonic-gate if (pp->cp_ncpus == 1) { 1823*0Sstevel@tonic-gate cp_numparts_nonempty++; 1824*0Sstevel@tonic-gate ASSERT(cp_numparts_nonempty != 0); 1825*0Sstevel@tonic-gate } 1826*0Sstevel@tonic-gate 1827*0Sstevel@tonic-gate chip_cpu_assign(cp); 1828*0Sstevel@tonic-gate 1829*0Sstevel@tonic-gate lgrp_config(LGRP_CONFIG_CPU_ONLINE, (uintptr_t)cp, 0); 1830*0Sstevel@tonic-gate 1831*0Sstevel@tonic-gate bzero(&cp->cpu_loadavg, sizeof (cp->cpu_loadavg)); 1832*0Sstevel@tonic-gate } 1833*0Sstevel@tonic-gate 1834*0Sstevel@tonic-gate /* 1835*0Sstevel@tonic-gate * Add a CPU to the list of active CPUs. 1836*0Sstevel@tonic-gate * This is called from machine-dependent layers when a new CPU is started. 1837*0Sstevel@tonic-gate */ 1838*0Sstevel@tonic-gate void 1839*0Sstevel@tonic-gate cpu_add_active(cpu_t *cp) 1840*0Sstevel@tonic-gate { 1841*0Sstevel@tonic-gate pause_cpus(NULL); 1842*0Sstevel@tonic-gate cpu_add_active_internal(cp); 1843*0Sstevel@tonic-gate start_cpus(); 1844*0Sstevel@tonic-gate cpu_stats_kstat_create(cp); 1845*0Sstevel@tonic-gate cpu_create_intrstat(cp); 1846*0Sstevel@tonic-gate lgrp_kstat_create(cp); 1847*0Sstevel@tonic-gate cpu_state_change_notify(cp->cpu_id, CPU_INIT); 1848*0Sstevel@tonic-gate } 1849*0Sstevel@tonic-gate 1850*0Sstevel@tonic-gate 1851*0Sstevel@tonic-gate /* 1852*0Sstevel@tonic-gate * Remove a CPU from the list of active CPUs. 1853*0Sstevel@tonic-gate * This routine must not get any locks, because other CPUs are paused. 1854*0Sstevel@tonic-gate */ 1855*0Sstevel@tonic-gate /* ARGSUSED */ 1856*0Sstevel@tonic-gate static void 1857*0Sstevel@tonic-gate cpu_remove_active(cpu_t *cp) 1858*0Sstevel@tonic-gate { 1859*0Sstevel@tonic-gate cpupart_t *pp = cp->cpu_part; 1860*0Sstevel@tonic-gate 1861*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1862*0Sstevel@tonic-gate ASSERT(cp->cpu_next_onln != cp); /* not the last one */ 1863*0Sstevel@tonic-gate ASSERT(cp->cpu_prev_onln != cp); /* not the last one */ 1864*0Sstevel@tonic-gate 1865*0Sstevel@tonic-gate chip_cpu_unassign(cp); 1866*0Sstevel@tonic-gate 1867*0Sstevel@tonic-gate lgrp_config(LGRP_CONFIG_CPU_OFFLINE, (uintptr_t)cp, 0); 1868*0Sstevel@tonic-gate 1869*0Sstevel@tonic-gate cp->cpu_prev_onln->cpu_next_onln = cp->cpu_next_onln; 1870*0Sstevel@tonic-gate cp->cpu_next_onln->cpu_prev_onln = cp->cpu_prev_onln; 1871*0Sstevel@tonic-gate if (cpu_active == cp) { 1872*0Sstevel@tonic-gate cpu_active = cp->cpu_next_onln; 1873*0Sstevel@tonic-gate } 1874*0Sstevel@tonic-gate cp->cpu_next_onln = cp; 1875*0Sstevel@tonic-gate cp->cpu_prev_onln = cp; 1876*0Sstevel@tonic-gate 1877*0Sstevel@tonic-gate cp->cpu_prev_part->cpu_next_part = cp->cpu_next_part; 1878*0Sstevel@tonic-gate cp->cpu_next_part->cpu_prev_part = cp->cpu_prev_part; 1879*0Sstevel@tonic-gate if (pp->cp_cpulist == cp) { 1880*0Sstevel@tonic-gate pp->cp_cpulist = cp->cpu_next_part; 1881*0Sstevel@tonic-gate ASSERT(pp->cp_cpulist != cp); 1882*0Sstevel@tonic-gate } 1883*0Sstevel@tonic-gate cp->cpu_next_part = cp; 1884*0Sstevel@tonic-gate cp->cpu_prev_part = cp; 1885*0Sstevel@tonic-gate pp->cp_ncpus--; 1886*0Sstevel@tonic-gate if (pp->cp_ncpus == 0) { 1887*0Sstevel@tonic-gate cp_numparts_nonempty--; 1888*0Sstevel@tonic-gate ASSERT(cp_numparts_nonempty != 0); 1889*0Sstevel@tonic-gate } 1890*0Sstevel@tonic-gate } 1891*0Sstevel@tonic-gate 1892*0Sstevel@tonic-gate /* 1893*0Sstevel@tonic-gate * Routine used to setup a newly inserted CPU in preparation for starting 1894*0Sstevel@tonic-gate * it running code. 1895*0Sstevel@tonic-gate */ 1896*0Sstevel@tonic-gate int 1897*0Sstevel@tonic-gate cpu_configure(int cpuid) 1898*0Sstevel@tonic-gate { 1899*0Sstevel@tonic-gate int retval = 0; 1900*0Sstevel@tonic-gate 1901*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1902*0Sstevel@tonic-gate 1903*0Sstevel@tonic-gate /* 1904*0Sstevel@tonic-gate * Some structures are statically allocated based upon 1905*0Sstevel@tonic-gate * the maximum number of cpus the system supports. Do not 1906*0Sstevel@tonic-gate * try to add anything beyond this limit. 1907*0Sstevel@tonic-gate */ 1908*0Sstevel@tonic-gate if (cpuid < 0 || cpuid >= NCPU) { 1909*0Sstevel@tonic-gate return (EINVAL); 1910*0Sstevel@tonic-gate } 1911*0Sstevel@tonic-gate 1912*0Sstevel@tonic-gate if ((cpu[cpuid] != NULL) && (cpu[cpuid]->cpu_flags != 0)) { 1913*0Sstevel@tonic-gate return (EALREADY); 1914*0Sstevel@tonic-gate } 1915*0Sstevel@tonic-gate 1916*0Sstevel@tonic-gate if ((retval = mp_cpu_configure(cpuid)) != 0) { 1917*0Sstevel@tonic-gate return (retval); 1918*0Sstevel@tonic-gate } 1919*0Sstevel@tonic-gate 1920*0Sstevel@tonic-gate cpu[cpuid]->cpu_flags = CPU_QUIESCED | CPU_OFFLINE | CPU_POWEROFF; 1921*0Sstevel@tonic-gate cpu_set_state(cpu[cpuid]); 1922*0Sstevel@tonic-gate retval = cpu_state_change_hooks(cpuid, CPU_CONFIG, CPU_UNCONFIG); 1923*0Sstevel@tonic-gate if (retval != 0) 1924*0Sstevel@tonic-gate (void) mp_cpu_unconfigure(cpuid); 1925*0Sstevel@tonic-gate 1926*0Sstevel@tonic-gate return (retval); 1927*0Sstevel@tonic-gate } 1928*0Sstevel@tonic-gate 1929*0Sstevel@tonic-gate /* 1930*0Sstevel@tonic-gate * Routine used to cleanup a CPU that has been powered off. This will 1931*0Sstevel@tonic-gate * destroy all per-cpu information related to this cpu. 1932*0Sstevel@tonic-gate */ 1933*0Sstevel@tonic-gate int 1934*0Sstevel@tonic-gate cpu_unconfigure(int cpuid) 1935*0Sstevel@tonic-gate { 1936*0Sstevel@tonic-gate int error; 1937*0Sstevel@tonic-gate 1938*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1939*0Sstevel@tonic-gate 1940*0Sstevel@tonic-gate if (cpu[cpuid] == NULL) { 1941*0Sstevel@tonic-gate return (ENODEV); 1942*0Sstevel@tonic-gate } 1943*0Sstevel@tonic-gate 1944*0Sstevel@tonic-gate if (cpu[cpuid]->cpu_flags == 0) { 1945*0Sstevel@tonic-gate return (EALREADY); 1946*0Sstevel@tonic-gate } 1947*0Sstevel@tonic-gate 1948*0Sstevel@tonic-gate if ((cpu[cpuid]->cpu_flags & CPU_POWEROFF) == 0) { 1949*0Sstevel@tonic-gate return (EBUSY); 1950*0Sstevel@tonic-gate } 1951*0Sstevel@tonic-gate 1952*0Sstevel@tonic-gate if (cpu[cpuid]->cpu_props != NULL) { 1953*0Sstevel@tonic-gate (void) nvlist_free(cpu[cpuid]->cpu_props); 1954*0Sstevel@tonic-gate cpu[cpuid]->cpu_props = NULL; 1955*0Sstevel@tonic-gate } 1956*0Sstevel@tonic-gate 1957*0Sstevel@tonic-gate error = cpu_state_change_hooks(cpuid, CPU_UNCONFIG, CPU_CONFIG); 1958*0Sstevel@tonic-gate 1959*0Sstevel@tonic-gate if (error != 0) 1960*0Sstevel@tonic-gate return (error); 1961*0Sstevel@tonic-gate 1962*0Sstevel@tonic-gate return (mp_cpu_unconfigure(cpuid)); 1963*0Sstevel@tonic-gate } 1964*0Sstevel@tonic-gate 1965*0Sstevel@tonic-gate /* 1966*0Sstevel@tonic-gate * Routines for registering and de-registering cpu_setup callback functions. 1967*0Sstevel@tonic-gate * 1968*0Sstevel@tonic-gate * Caller's context 1969*0Sstevel@tonic-gate * These routines must not be called from a driver's attach(9E) or 1970*0Sstevel@tonic-gate * detach(9E) entry point. 1971*0Sstevel@tonic-gate * 1972*0Sstevel@tonic-gate * NOTE: CPU callbacks should not block. They are called with cpu_lock held. 1973*0Sstevel@tonic-gate */ 1974*0Sstevel@tonic-gate 1975*0Sstevel@tonic-gate /* 1976*0Sstevel@tonic-gate * Ideally, these would be dynamically allocated and put into a linked 1977*0Sstevel@tonic-gate * list; however that is not feasible because the registration routine 1978*0Sstevel@tonic-gate * has to be available before the kmem allocator is working (in fact, 1979*0Sstevel@tonic-gate * it is called by the kmem allocator init code). In any case, there 1980*0Sstevel@tonic-gate * are quite a few extra entries for future users. 1981*0Sstevel@tonic-gate */ 1982*0Sstevel@tonic-gate #define NCPU_SETUPS 10 1983*0Sstevel@tonic-gate 1984*0Sstevel@tonic-gate struct cpu_setup { 1985*0Sstevel@tonic-gate cpu_setup_func_t *func; 1986*0Sstevel@tonic-gate void *arg; 1987*0Sstevel@tonic-gate } cpu_setups[NCPU_SETUPS]; 1988*0Sstevel@tonic-gate 1989*0Sstevel@tonic-gate void 1990*0Sstevel@tonic-gate register_cpu_setup_func(cpu_setup_func_t *func, void *arg) 1991*0Sstevel@tonic-gate { 1992*0Sstevel@tonic-gate int i; 1993*0Sstevel@tonic-gate 1994*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1995*0Sstevel@tonic-gate 1996*0Sstevel@tonic-gate for (i = 0; i < NCPU_SETUPS; i++) 1997*0Sstevel@tonic-gate if (cpu_setups[i].func == NULL) 1998*0Sstevel@tonic-gate break; 1999*0Sstevel@tonic-gate if (i >= NCPU_SETUPS) 2000*0Sstevel@tonic-gate cmn_err(CE_PANIC, "Ran out of cpu_setup callback entries"); 2001*0Sstevel@tonic-gate 2002*0Sstevel@tonic-gate cpu_setups[i].func = func; 2003*0Sstevel@tonic-gate cpu_setups[i].arg = arg; 2004*0Sstevel@tonic-gate } 2005*0Sstevel@tonic-gate 2006*0Sstevel@tonic-gate void 2007*0Sstevel@tonic-gate unregister_cpu_setup_func(cpu_setup_func_t *func, void *arg) 2008*0Sstevel@tonic-gate { 2009*0Sstevel@tonic-gate int i; 2010*0Sstevel@tonic-gate 2011*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 2012*0Sstevel@tonic-gate 2013*0Sstevel@tonic-gate for (i = 0; i < NCPU_SETUPS; i++) 2014*0Sstevel@tonic-gate if ((cpu_setups[i].func == func) && 2015*0Sstevel@tonic-gate (cpu_setups[i].arg == arg)) 2016*0Sstevel@tonic-gate break; 2017*0Sstevel@tonic-gate if (i >= NCPU_SETUPS) 2018*0Sstevel@tonic-gate cmn_err(CE_PANIC, "Could not find cpu_setup callback to " 2019*0Sstevel@tonic-gate "deregister"); 2020*0Sstevel@tonic-gate 2021*0Sstevel@tonic-gate cpu_setups[i].func = NULL; 2022*0Sstevel@tonic-gate cpu_setups[i].arg = 0; 2023*0Sstevel@tonic-gate } 2024*0Sstevel@tonic-gate 2025*0Sstevel@tonic-gate /* 2026*0Sstevel@tonic-gate * Call any state change hooks for this CPU, ignore any errors. 2027*0Sstevel@tonic-gate */ 2028*0Sstevel@tonic-gate void 2029*0Sstevel@tonic-gate cpu_state_change_notify(int id, cpu_setup_t what) 2030*0Sstevel@tonic-gate { 2031*0Sstevel@tonic-gate int i; 2032*0Sstevel@tonic-gate 2033*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 2034*0Sstevel@tonic-gate 2035*0Sstevel@tonic-gate for (i = 0; i < NCPU_SETUPS; i++) { 2036*0Sstevel@tonic-gate if (cpu_setups[i].func != NULL) { 2037*0Sstevel@tonic-gate cpu_setups[i].func(what, id, cpu_setups[i].arg); 2038*0Sstevel@tonic-gate } 2039*0Sstevel@tonic-gate } 2040*0Sstevel@tonic-gate } 2041*0Sstevel@tonic-gate 2042*0Sstevel@tonic-gate /* 2043*0Sstevel@tonic-gate * Call any state change hooks for this CPU, undo it if error found. 2044*0Sstevel@tonic-gate */ 2045*0Sstevel@tonic-gate static int 2046*0Sstevel@tonic-gate cpu_state_change_hooks(int id, cpu_setup_t what, cpu_setup_t undo) 2047*0Sstevel@tonic-gate { 2048*0Sstevel@tonic-gate int i; 2049*0Sstevel@tonic-gate int retval = 0; 2050*0Sstevel@tonic-gate 2051*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 2052*0Sstevel@tonic-gate 2053*0Sstevel@tonic-gate for (i = 0; i < NCPU_SETUPS; i++) { 2054*0Sstevel@tonic-gate if (cpu_setups[i].func != NULL) { 2055*0Sstevel@tonic-gate retval = cpu_setups[i].func(what, id, 2056*0Sstevel@tonic-gate cpu_setups[i].arg); 2057*0Sstevel@tonic-gate if (retval) { 2058*0Sstevel@tonic-gate for (i--; i >= 0; i--) { 2059*0Sstevel@tonic-gate if (cpu_setups[i].func != NULL) 2060*0Sstevel@tonic-gate cpu_setups[i].func(undo, 2061*0Sstevel@tonic-gate id, cpu_setups[i].arg); 2062*0Sstevel@tonic-gate } 2063*0Sstevel@tonic-gate break; 2064*0Sstevel@tonic-gate } 2065*0Sstevel@tonic-gate } 2066*0Sstevel@tonic-gate } 2067*0Sstevel@tonic-gate return (retval); 2068*0Sstevel@tonic-gate } 2069*0Sstevel@tonic-gate 2070*0Sstevel@tonic-gate /* 2071*0Sstevel@tonic-gate * Export information about this CPU via the kstat mechanism. 2072*0Sstevel@tonic-gate */ 2073*0Sstevel@tonic-gate static struct { 2074*0Sstevel@tonic-gate kstat_named_t ci_state; 2075*0Sstevel@tonic-gate kstat_named_t ci_state_begin; 2076*0Sstevel@tonic-gate kstat_named_t ci_cpu_type; 2077*0Sstevel@tonic-gate kstat_named_t ci_fpu_type; 2078*0Sstevel@tonic-gate kstat_named_t ci_clock_MHz; 2079*0Sstevel@tonic-gate kstat_named_t ci_chip_id; 2080*0Sstevel@tonic-gate kstat_named_t ci_implementation; 2081*0Sstevel@tonic-gate #ifdef __sparcv9 2082*0Sstevel@tonic-gate kstat_named_t ci_device_ID; 2083*0Sstevel@tonic-gate kstat_named_t ci_cpu_fru; 2084*0Sstevel@tonic-gate #endif 2085*0Sstevel@tonic-gate kstat_named_t ci_brandstr; 2086*0Sstevel@tonic-gate } cpu_info_template = { 2087*0Sstevel@tonic-gate { "state", KSTAT_DATA_CHAR }, 2088*0Sstevel@tonic-gate { "state_begin", KSTAT_DATA_LONG }, 2089*0Sstevel@tonic-gate { "cpu_type", KSTAT_DATA_CHAR }, 2090*0Sstevel@tonic-gate { "fpu_type", KSTAT_DATA_CHAR }, 2091*0Sstevel@tonic-gate { "clock_MHz", KSTAT_DATA_LONG }, 2092*0Sstevel@tonic-gate { "chip_id", KSTAT_DATA_LONG }, 2093*0Sstevel@tonic-gate { "implementation", KSTAT_DATA_STRING }, 2094*0Sstevel@tonic-gate #ifdef __sparcv9 2095*0Sstevel@tonic-gate { "device_ID", KSTAT_DATA_UINT64 }, 2096*0Sstevel@tonic-gate { "cpu_fru", KSTAT_DATA_STRING }, 2097*0Sstevel@tonic-gate #endif 2098*0Sstevel@tonic-gate { "brand", KSTAT_DATA_STRING }, 2099*0Sstevel@tonic-gate }; 2100*0Sstevel@tonic-gate 2101*0Sstevel@tonic-gate static kmutex_t cpu_info_template_lock; 2102*0Sstevel@tonic-gate 2103*0Sstevel@tonic-gate static int 2104*0Sstevel@tonic-gate cpu_info_kstat_update(kstat_t *ksp, int rw) 2105*0Sstevel@tonic-gate { 2106*0Sstevel@tonic-gate cpu_t *cp = ksp->ks_private; 2107*0Sstevel@tonic-gate const char *pi_state; 2108*0Sstevel@tonic-gate 2109*0Sstevel@tonic-gate if (rw == KSTAT_WRITE) 2110*0Sstevel@tonic-gate return (EACCES); 2111*0Sstevel@tonic-gate 2112*0Sstevel@tonic-gate switch (cp->cpu_type_info.pi_state) { 2113*0Sstevel@tonic-gate case P_ONLINE: 2114*0Sstevel@tonic-gate pi_state = PS_ONLINE; 2115*0Sstevel@tonic-gate break; 2116*0Sstevel@tonic-gate case P_POWEROFF: 2117*0Sstevel@tonic-gate pi_state = PS_POWEROFF; 2118*0Sstevel@tonic-gate break; 2119*0Sstevel@tonic-gate case P_NOINTR: 2120*0Sstevel@tonic-gate pi_state = PS_NOINTR; 2121*0Sstevel@tonic-gate break; 2122*0Sstevel@tonic-gate case P_FAULTED: 2123*0Sstevel@tonic-gate pi_state = PS_FAULTED; 2124*0Sstevel@tonic-gate break; 2125*0Sstevel@tonic-gate case P_SPARE: 2126*0Sstevel@tonic-gate pi_state = PS_SPARE; 2127*0Sstevel@tonic-gate break; 2128*0Sstevel@tonic-gate case P_OFFLINE: 2129*0Sstevel@tonic-gate pi_state = PS_OFFLINE; 2130*0Sstevel@tonic-gate break; 2131*0Sstevel@tonic-gate default: 2132*0Sstevel@tonic-gate pi_state = "unknown"; 2133*0Sstevel@tonic-gate } 2134*0Sstevel@tonic-gate (void) strcpy(cpu_info_template.ci_state.value.c, pi_state); 2135*0Sstevel@tonic-gate cpu_info_template.ci_state_begin.value.l = cp->cpu_state_begin; 2136*0Sstevel@tonic-gate (void) strncpy(cpu_info_template.ci_cpu_type.value.c, 2137*0Sstevel@tonic-gate cp->cpu_type_info.pi_processor_type, 15); 2138*0Sstevel@tonic-gate (void) strncpy(cpu_info_template.ci_fpu_type.value.c, 2139*0Sstevel@tonic-gate cp->cpu_type_info.pi_fputypes, 15); 2140*0Sstevel@tonic-gate cpu_info_template.ci_clock_MHz.value.l = cp->cpu_type_info.pi_clock; 2141*0Sstevel@tonic-gate cpu_info_template.ci_chip_id.value.l = chip_plat_get_chipid(cp); 2142*0Sstevel@tonic-gate kstat_named_setstr(&cpu_info_template.ci_implementation, 2143*0Sstevel@tonic-gate cp->cpu_idstr); 2144*0Sstevel@tonic-gate #ifdef __sparcv9 2145*0Sstevel@tonic-gate cpu_info_template.ci_device_ID.value.ui64 = 2146*0Sstevel@tonic-gate cpunodes[cp->cpu_id].device_id; 2147*0Sstevel@tonic-gate kstat_named_setstr(&cpu_info_template.ci_cpu_fru, cpu_fru_fmri(cp)); 2148*0Sstevel@tonic-gate #endif 2149*0Sstevel@tonic-gate kstat_named_setstr(&cpu_info_template.ci_brandstr, cp->cpu_brandstr); 2150*0Sstevel@tonic-gate return (0); 2151*0Sstevel@tonic-gate } 2152*0Sstevel@tonic-gate 2153*0Sstevel@tonic-gate static void 2154*0Sstevel@tonic-gate cpu_info_kstat_create(cpu_t *cp) 2155*0Sstevel@tonic-gate { 2156*0Sstevel@tonic-gate zoneid_t zoneid; 2157*0Sstevel@tonic-gate 2158*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 2159*0Sstevel@tonic-gate 2160*0Sstevel@tonic-gate if (pool_pset_enabled()) 2161*0Sstevel@tonic-gate zoneid = GLOBAL_ZONEID; 2162*0Sstevel@tonic-gate else 2163*0Sstevel@tonic-gate zoneid = ALL_ZONES; 2164*0Sstevel@tonic-gate if ((cp->cpu_info_kstat = kstat_create_zone("cpu_info", cp->cpu_id, 2165*0Sstevel@tonic-gate NULL, "misc", KSTAT_TYPE_NAMED, 2166*0Sstevel@tonic-gate sizeof (cpu_info_template) / sizeof (kstat_named_t), 2167*0Sstevel@tonic-gate KSTAT_FLAG_VIRTUAL, zoneid)) != NULL) { 2168*0Sstevel@tonic-gate cp->cpu_info_kstat->ks_data_size += 2 * CPU_IDSTRLEN; 2169*0Sstevel@tonic-gate #ifdef __sparcv9 2170*0Sstevel@tonic-gate cp->cpu_info_kstat->ks_data_size += 2171*0Sstevel@tonic-gate strlen(cpu_fru_fmri(cp)) + 1; 2172*0Sstevel@tonic-gate #endif 2173*0Sstevel@tonic-gate cp->cpu_info_kstat->ks_lock = &cpu_info_template_lock; 2174*0Sstevel@tonic-gate cp->cpu_info_kstat->ks_data = &cpu_info_template; 2175*0Sstevel@tonic-gate cp->cpu_info_kstat->ks_private = cp; 2176*0Sstevel@tonic-gate cp->cpu_info_kstat->ks_update = cpu_info_kstat_update; 2177*0Sstevel@tonic-gate kstat_install(cp->cpu_info_kstat); 2178*0Sstevel@tonic-gate } 2179*0Sstevel@tonic-gate } 2180*0Sstevel@tonic-gate 2181*0Sstevel@tonic-gate static void 2182*0Sstevel@tonic-gate cpu_info_kstat_destroy(cpu_t *cp) 2183*0Sstevel@tonic-gate { 2184*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 2185*0Sstevel@tonic-gate 2186*0Sstevel@tonic-gate kstat_delete(cp->cpu_info_kstat); 2187*0Sstevel@tonic-gate cp->cpu_info_kstat = NULL; 2188*0Sstevel@tonic-gate } 2189*0Sstevel@tonic-gate 2190*0Sstevel@tonic-gate /* 2191*0Sstevel@tonic-gate * Create and install kstats for the boot CPU. 2192*0Sstevel@tonic-gate */ 2193*0Sstevel@tonic-gate void 2194*0Sstevel@tonic-gate cpu_kstat_init(cpu_t *cp) 2195*0Sstevel@tonic-gate { 2196*0Sstevel@tonic-gate mutex_enter(&cpu_lock); 2197*0Sstevel@tonic-gate cpu_info_kstat_create(cp); 2198*0Sstevel@tonic-gate cpu_stats_kstat_create(cp); 2199*0Sstevel@tonic-gate cpu_create_intrstat(cp); 2200*0Sstevel@tonic-gate chip_kstat_create(cp->cpu_chip); 2201*0Sstevel@tonic-gate cpu_set_state(cp); 2202*0Sstevel@tonic-gate mutex_exit(&cpu_lock); 2203*0Sstevel@tonic-gate } 2204*0Sstevel@tonic-gate 2205*0Sstevel@tonic-gate /* 2206*0Sstevel@tonic-gate * Make visible to the zone that subset of the cpu information that would be 2207*0Sstevel@tonic-gate * initialized when a cpu is configured (but still offline). 2208*0Sstevel@tonic-gate */ 2209*0Sstevel@tonic-gate void 2210*0Sstevel@tonic-gate cpu_visibility_configure(cpu_t *cp, zone_t *zone) 2211*0Sstevel@tonic-gate { 2212*0Sstevel@tonic-gate zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES; 2213*0Sstevel@tonic-gate 2214*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 2215*0Sstevel@tonic-gate ASSERT(pool_pset_enabled()); 2216*0Sstevel@tonic-gate ASSERT(cp != NULL); 2217*0Sstevel@tonic-gate 2218*0Sstevel@tonic-gate if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) { 2219*0Sstevel@tonic-gate zone->zone_ncpus++; 2220*0Sstevel@tonic-gate ASSERT(zone->zone_ncpus <= ncpus); 2221*0Sstevel@tonic-gate } 2222*0Sstevel@tonic-gate if (cp->cpu_info_kstat != NULL) 2223*0Sstevel@tonic-gate kstat_zone_add(cp->cpu_info_kstat, zoneid); 2224*0Sstevel@tonic-gate } 2225*0Sstevel@tonic-gate 2226*0Sstevel@tonic-gate /* 2227*0Sstevel@tonic-gate * Make visible to the zone that subset of the cpu information that would be 2228*0Sstevel@tonic-gate * initialized when a previously configured cpu is onlined. 2229*0Sstevel@tonic-gate */ 2230*0Sstevel@tonic-gate void 2231*0Sstevel@tonic-gate cpu_visibility_online(cpu_t *cp, zone_t *zone) 2232*0Sstevel@tonic-gate { 2233*0Sstevel@tonic-gate kstat_t *ksp; 2234*0Sstevel@tonic-gate char name[sizeof ("cpu_stat") + 10]; /* enough for 32-bit cpuids */ 2235*0Sstevel@tonic-gate zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES; 2236*0Sstevel@tonic-gate processorid_t cpun; 2237*0Sstevel@tonic-gate 2238*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 2239*0Sstevel@tonic-gate ASSERT(pool_pset_enabled()); 2240*0Sstevel@tonic-gate ASSERT(cp != NULL); 2241*0Sstevel@tonic-gate ASSERT(cpu_is_active(cp)); 2242*0Sstevel@tonic-gate 2243*0Sstevel@tonic-gate cpun = cp->cpu_id; 2244*0Sstevel@tonic-gate if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) { 2245*0Sstevel@tonic-gate zone->zone_ncpus_online++; 2246*0Sstevel@tonic-gate ASSERT(zone->zone_ncpus_online <= ncpus_online); 2247*0Sstevel@tonic-gate } 2248*0Sstevel@tonic-gate (void) snprintf(name, sizeof (name), "cpu_stat%d", cpun); 2249*0Sstevel@tonic-gate if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES)) 2250*0Sstevel@tonic-gate != NULL) { 2251*0Sstevel@tonic-gate kstat_zone_add(ksp, zoneid); 2252*0Sstevel@tonic-gate kstat_rele(ksp); 2253*0Sstevel@tonic-gate } 2254*0Sstevel@tonic-gate if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) { 2255*0Sstevel@tonic-gate kstat_zone_add(ksp, zoneid); 2256*0Sstevel@tonic-gate kstat_rele(ksp); 2257*0Sstevel@tonic-gate } 2258*0Sstevel@tonic-gate if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) { 2259*0Sstevel@tonic-gate kstat_zone_add(ksp, zoneid); 2260*0Sstevel@tonic-gate kstat_rele(ksp); 2261*0Sstevel@tonic-gate } 2262*0Sstevel@tonic-gate if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) != 2263*0Sstevel@tonic-gate NULL) { 2264*0Sstevel@tonic-gate kstat_zone_add(ksp, zoneid); 2265*0Sstevel@tonic-gate kstat_rele(ksp); 2266*0Sstevel@tonic-gate } 2267*0Sstevel@tonic-gate } 2268*0Sstevel@tonic-gate 2269*0Sstevel@tonic-gate /* 2270*0Sstevel@tonic-gate * Update relevant kstats such that cpu is now visible to processes 2271*0Sstevel@tonic-gate * executing in specified zone. 2272*0Sstevel@tonic-gate */ 2273*0Sstevel@tonic-gate void 2274*0Sstevel@tonic-gate cpu_visibility_add(cpu_t *cp, zone_t *zone) 2275*0Sstevel@tonic-gate { 2276*0Sstevel@tonic-gate cpu_visibility_configure(cp, zone); 2277*0Sstevel@tonic-gate if (cpu_is_active(cp)) 2278*0Sstevel@tonic-gate cpu_visibility_online(cp, zone); 2279*0Sstevel@tonic-gate } 2280*0Sstevel@tonic-gate 2281*0Sstevel@tonic-gate /* 2282*0Sstevel@tonic-gate * Make invisible to the zone that subset of the cpu information that would be 2283*0Sstevel@tonic-gate * torn down when a previously offlined cpu is unconfigured. 2284*0Sstevel@tonic-gate */ 2285*0Sstevel@tonic-gate void 2286*0Sstevel@tonic-gate cpu_visibility_unconfigure(cpu_t *cp, zone_t *zone) 2287*0Sstevel@tonic-gate { 2288*0Sstevel@tonic-gate zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES; 2289*0Sstevel@tonic-gate 2290*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 2291*0Sstevel@tonic-gate ASSERT(pool_pset_enabled()); 2292*0Sstevel@tonic-gate ASSERT(cp != NULL); 2293*0Sstevel@tonic-gate 2294*0Sstevel@tonic-gate if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) { 2295*0Sstevel@tonic-gate ASSERT(zone->zone_ncpus != 0); 2296*0Sstevel@tonic-gate zone->zone_ncpus--; 2297*0Sstevel@tonic-gate } 2298*0Sstevel@tonic-gate if (cp->cpu_info_kstat) 2299*0Sstevel@tonic-gate kstat_zone_remove(cp->cpu_info_kstat, zoneid); 2300*0Sstevel@tonic-gate } 2301*0Sstevel@tonic-gate 2302*0Sstevel@tonic-gate /* 2303*0Sstevel@tonic-gate * Make invisible to the zone that subset of the cpu information that would be 2304*0Sstevel@tonic-gate * torn down when a cpu is offlined (but still configured). 2305*0Sstevel@tonic-gate */ 2306*0Sstevel@tonic-gate void 2307*0Sstevel@tonic-gate cpu_visibility_offline(cpu_t *cp, zone_t *zone) 2308*0Sstevel@tonic-gate { 2309*0Sstevel@tonic-gate kstat_t *ksp; 2310*0Sstevel@tonic-gate char name[sizeof ("cpu_stat") + 10]; /* enough for 32-bit cpuids */ 2311*0Sstevel@tonic-gate zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES; 2312*0Sstevel@tonic-gate processorid_t cpun; 2313*0Sstevel@tonic-gate 2314*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 2315*0Sstevel@tonic-gate ASSERT(pool_pset_enabled()); 2316*0Sstevel@tonic-gate ASSERT(cp != NULL); 2317*0Sstevel@tonic-gate ASSERT(cpu_is_active(cp)); 2318*0Sstevel@tonic-gate 2319*0Sstevel@tonic-gate cpun = cp->cpu_id; 2320*0Sstevel@tonic-gate if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) { 2321*0Sstevel@tonic-gate ASSERT(zone->zone_ncpus_online != 0); 2322*0Sstevel@tonic-gate zone->zone_ncpus_online--; 2323*0Sstevel@tonic-gate } 2324*0Sstevel@tonic-gate 2325*0Sstevel@tonic-gate if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) != 2326*0Sstevel@tonic-gate NULL) { 2327*0Sstevel@tonic-gate kstat_zone_remove(ksp, zoneid); 2328*0Sstevel@tonic-gate kstat_rele(ksp); 2329*0Sstevel@tonic-gate } 2330*0Sstevel@tonic-gate if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) { 2331*0Sstevel@tonic-gate kstat_zone_remove(ksp, zoneid); 2332*0Sstevel@tonic-gate kstat_rele(ksp); 2333*0Sstevel@tonic-gate } 2334*0Sstevel@tonic-gate if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) { 2335*0Sstevel@tonic-gate kstat_zone_remove(ksp, zoneid); 2336*0Sstevel@tonic-gate kstat_rele(ksp); 2337*0Sstevel@tonic-gate } 2338*0Sstevel@tonic-gate (void) snprintf(name, sizeof (name), "cpu_stat%d", cpun); 2339*0Sstevel@tonic-gate if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES)) 2340*0Sstevel@tonic-gate != NULL) { 2341*0Sstevel@tonic-gate kstat_zone_remove(ksp, zoneid); 2342*0Sstevel@tonic-gate kstat_rele(ksp); 2343*0Sstevel@tonic-gate } 2344*0Sstevel@tonic-gate } 2345*0Sstevel@tonic-gate 2346*0Sstevel@tonic-gate /* 2347*0Sstevel@tonic-gate * Update relevant kstats such that cpu is no longer visible to processes 2348*0Sstevel@tonic-gate * executing in specified zone. 2349*0Sstevel@tonic-gate */ 2350*0Sstevel@tonic-gate void 2351*0Sstevel@tonic-gate cpu_visibility_remove(cpu_t *cp, zone_t *zone) 2352*0Sstevel@tonic-gate { 2353*0Sstevel@tonic-gate if (cpu_is_active(cp)) 2354*0Sstevel@tonic-gate cpu_visibility_offline(cp, zone); 2355*0Sstevel@tonic-gate cpu_visibility_unconfigure(cp, zone); 2356*0Sstevel@tonic-gate } 2357*0Sstevel@tonic-gate 2358*0Sstevel@tonic-gate /* 2359*0Sstevel@tonic-gate * Bind a thread to a CPU as requested. 2360*0Sstevel@tonic-gate */ 2361*0Sstevel@tonic-gate int 2362*0Sstevel@tonic-gate cpu_bind_thread(kthread_id_t tp, processorid_t bind, processorid_t *obind, 2363*0Sstevel@tonic-gate int *error) 2364*0Sstevel@tonic-gate { 2365*0Sstevel@tonic-gate processorid_t binding; 2366*0Sstevel@tonic-gate cpu_t *cp; 2367*0Sstevel@tonic-gate 2368*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 2369*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ttoproc(tp)->p_lock)); 2370*0Sstevel@tonic-gate 2371*0Sstevel@tonic-gate thread_lock(tp); 2372*0Sstevel@tonic-gate 2373*0Sstevel@tonic-gate /* 2374*0Sstevel@tonic-gate * Record old binding, but change the obind, which was initialized 2375*0Sstevel@tonic-gate * to PBIND_NONE, only if this thread has a binding. This avoids 2376*0Sstevel@tonic-gate * reporting PBIND_NONE for a process when some LWPs are bound. 2377*0Sstevel@tonic-gate */ 2378*0Sstevel@tonic-gate binding = tp->t_bind_cpu; 2379*0Sstevel@tonic-gate if (binding != PBIND_NONE) 2380*0Sstevel@tonic-gate *obind = binding; /* record old binding */ 2381*0Sstevel@tonic-gate 2382*0Sstevel@tonic-gate if (bind == PBIND_QUERY) { 2383*0Sstevel@tonic-gate thread_unlock(tp); 2384*0Sstevel@tonic-gate return (0); 2385*0Sstevel@tonic-gate } 2386*0Sstevel@tonic-gate 2387*0Sstevel@tonic-gate /* 2388*0Sstevel@tonic-gate * If this thread/LWP cannot be bound because of permission 2389*0Sstevel@tonic-gate * problems, just note that and return success so that the 2390*0Sstevel@tonic-gate * other threads/LWPs will be bound. This is the way 2391*0Sstevel@tonic-gate * processor_bind() is defined to work. 2392*0Sstevel@tonic-gate * 2393*0Sstevel@tonic-gate * Binding will get EPERM if the thread is of system class 2394*0Sstevel@tonic-gate * or hasprocperm() fails. 2395*0Sstevel@tonic-gate */ 2396*0Sstevel@tonic-gate if (tp->t_cid == 0 || !hasprocperm(tp->t_cred, CRED())) { 2397*0Sstevel@tonic-gate *error = EPERM; 2398*0Sstevel@tonic-gate thread_unlock(tp); 2399*0Sstevel@tonic-gate return (0); 2400*0Sstevel@tonic-gate } 2401*0Sstevel@tonic-gate 2402*0Sstevel@tonic-gate binding = bind; 2403*0Sstevel@tonic-gate if (binding != PBIND_NONE) { 2404*0Sstevel@tonic-gate cp = cpu[binding]; 2405*0Sstevel@tonic-gate /* 2406*0Sstevel@tonic-gate * Make sure binding is in right partition. 2407*0Sstevel@tonic-gate */ 2408*0Sstevel@tonic-gate if (tp->t_cpupart != cp->cpu_part) { 2409*0Sstevel@tonic-gate *error = EINVAL; 2410*0Sstevel@tonic-gate thread_unlock(tp); 2411*0Sstevel@tonic-gate return (0); 2412*0Sstevel@tonic-gate } 2413*0Sstevel@tonic-gate } 2414*0Sstevel@tonic-gate tp->t_bind_cpu = binding; /* set new binding */ 2415*0Sstevel@tonic-gate 2416*0Sstevel@tonic-gate /* 2417*0Sstevel@tonic-gate * If there is no system-set reason for affinity, set 2418*0Sstevel@tonic-gate * the t_bound_cpu field to reflect the binding. 2419*0Sstevel@tonic-gate */ 2420*0Sstevel@tonic-gate if (tp->t_affinitycnt == 0) { 2421*0Sstevel@tonic-gate if (binding == PBIND_NONE) { 2422*0Sstevel@tonic-gate /* 2423*0Sstevel@tonic-gate * We may need to adjust disp_max_unbound_pri 2424*0Sstevel@tonic-gate * since we're becoming unbound. 2425*0Sstevel@tonic-gate */ 2426*0Sstevel@tonic-gate disp_adjust_unbound_pri(tp); 2427*0Sstevel@tonic-gate 2428*0Sstevel@tonic-gate tp->t_bound_cpu = NULL; /* set new binding */ 2429*0Sstevel@tonic-gate 2430*0Sstevel@tonic-gate /* 2431*0Sstevel@tonic-gate * Move thread to lgroup with strongest affinity 2432*0Sstevel@tonic-gate * after unbinding 2433*0Sstevel@tonic-gate */ 2434*0Sstevel@tonic-gate if (tp->t_lgrp_affinity) 2435*0Sstevel@tonic-gate lgrp_move_thread(tp, 2436*0Sstevel@tonic-gate lgrp_choose(tp, tp->t_cpupart), 1); 2437*0Sstevel@tonic-gate 2438*0Sstevel@tonic-gate if (tp->t_state == TS_ONPROC && 2439*0Sstevel@tonic-gate tp->t_cpu->cpu_part != tp->t_cpupart) 2440*0Sstevel@tonic-gate cpu_surrender(tp); 2441*0Sstevel@tonic-gate } else { 2442*0Sstevel@tonic-gate lpl_t *lpl; 2443*0Sstevel@tonic-gate 2444*0Sstevel@tonic-gate tp->t_bound_cpu = cp; 2445*0Sstevel@tonic-gate ASSERT(cp->cpu_lpl != NULL); 2446*0Sstevel@tonic-gate 2447*0Sstevel@tonic-gate /* 2448*0Sstevel@tonic-gate * Set home to lgroup with most affinity containing CPU 2449*0Sstevel@tonic-gate * that thread is being bound or minimum bounding 2450*0Sstevel@tonic-gate * lgroup if no affinities set 2451*0Sstevel@tonic-gate */ 2452*0Sstevel@tonic-gate if (tp->t_lgrp_affinity) 2453*0Sstevel@tonic-gate lpl = lgrp_affinity_best(tp, tp->t_cpupart, 0); 2454*0Sstevel@tonic-gate else 2455*0Sstevel@tonic-gate lpl = cp->cpu_lpl; 2456*0Sstevel@tonic-gate 2457*0Sstevel@tonic-gate if (tp->t_lpl != lpl) { 2458*0Sstevel@tonic-gate /* can't grab cpu_lock */ 2459*0Sstevel@tonic-gate lgrp_move_thread(tp, lpl, 1); 2460*0Sstevel@tonic-gate } 2461*0Sstevel@tonic-gate 2462*0Sstevel@tonic-gate /* 2463*0Sstevel@tonic-gate * Make the thread switch to the bound CPU. 2464*0Sstevel@tonic-gate * If the thread is runnable, we need to 2465*0Sstevel@tonic-gate * requeue it even if t_cpu is already set 2466*0Sstevel@tonic-gate * to the right CPU, since it may be on a 2467*0Sstevel@tonic-gate * kpreempt queue and need to move to a local 2468*0Sstevel@tonic-gate * queue. We could check t_disp_queue to 2469*0Sstevel@tonic-gate * avoid unnecessary overhead if it's already 2470*0Sstevel@tonic-gate * on the right queue, but since this isn't 2471*0Sstevel@tonic-gate * a performance-critical operation it doesn't 2472*0Sstevel@tonic-gate * seem worth the extra code and complexity. 2473*0Sstevel@tonic-gate * 2474*0Sstevel@tonic-gate * If the thread is weakbound to the cpu then it will 2475*0Sstevel@tonic-gate * resist the new binding request until the weak 2476*0Sstevel@tonic-gate * binding drops. The cpu_surrender or requeueing 2477*0Sstevel@tonic-gate * below could be skipped in such cases (since it 2478*0Sstevel@tonic-gate * will have no effect), but that would require 2479*0Sstevel@tonic-gate * thread_allowmigrate to acquire thread_lock so 2480*0Sstevel@tonic-gate * we'll take the very occasional hit here instead. 2481*0Sstevel@tonic-gate */ 2482*0Sstevel@tonic-gate if (tp->t_state == TS_ONPROC) { 2483*0Sstevel@tonic-gate cpu_surrender(tp); 2484*0Sstevel@tonic-gate } else if (tp->t_state == TS_RUN) { 2485*0Sstevel@tonic-gate cpu_t *ocp = tp->t_cpu; 2486*0Sstevel@tonic-gate 2487*0Sstevel@tonic-gate (void) dispdeq(tp); 2488*0Sstevel@tonic-gate setbackdq(tp); 2489*0Sstevel@tonic-gate /* 2490*0Sstevel@tonic-gate * Either on the bound CPU's disp queue now, 2491*0Sstevel@tonic-gate * or swapped out or on the swap queue. 2492*0Sstevel@tonic-gate */ 2493*0Sstevel@tonic-gate ASSERT(tp->t_disp_queue == cp->cpu_disp || 2494*0Sstevel@tonic-gate tp->t_weakbound_cpu == ocp || 2495*0Sstevel@tonic-gate (tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ)) 2496*0Sstevel@tonic-gate != TS_LOAD); 2497*0Sstevel@tonic-gate } 2498*0Sstevel@tonic-gate } 2499*0Sstevel@tonic-gate } 2500*0Sstevel@tonic-gate 2501*0Sstevel@tonic-gate /* 2502*0Sstevel@tonic-gate * Our binding has changed; set TP_CHANGEBIND. 2503*0Sstevel@tonic-gate */ 2504*0Sstevel@tonic-gate tp->t_proc_flag |= TP_CHANGEBIND; 2505*0Sstevel@tonic-gate aston(tp); 2506*0Sstevel@tonic-gate 2507*0Sstevel@tonic-gate thread_unlock(tp); 2508*0Sstevel@tonic-gate 2509*0Sstevel@tonic-gate return (0); 2510*0Sstevel@tonic-gate } 2511*0Sstevel@tonic-gate 2512*0Sstevel@tonic-gate #if CPUSET_WORDS > 1 2513*0Sstevel@tonic-gate 2514*0Sstevel@tonic-gate /* 2515*0Sstevel@tonic-gate * Functions for implementing cpuset operations when a cpuset is more 2516*0Sstevel@tonic-gate * than one word. On platforms where a cpuset is a single word these 2517*0Sstevel@tonic-gate * are implemented as macros in cpuvar.h. 2518*0Sstevel@tonic-gate */ 2519*0Sstevel@tonic-gate 2520*0Sstevel@tonic-gate void 2521*0Sstevel@tonic-gate cpuset_all(cpuset_t *s) 2522*0Sstevel@tonic-gate { 2523*0Sstevel@tonic-gate int i; 2524*0Sstevel@tonic-gate 2525*0Sstevel@tonic-gate for (i = 0; i < CPUSET_WORDS; i++) 2526*0Sstevel@tonic-gate s->cpub[i] = ~0UL; 2527*0Sstevel@tonic-gate } 2528*0Sstevel@tonic-gate 2529*0Sstevel@tonic-gate void 2530*0Sstevel@tonic-gate cpuset_all_but(cpuset_t *s, uint_t cpu) 2531*0Sstevel@tonic-gate { 2532*0Sstevel@tonic-gate cpuset_all(s); 2533*0Sstevel@tonic-gate CPUSET_DEL(*s, cpu); 2534*0Sstevel@tonic-gate } 2535*0Sstevel@tonic-gate 2536*0Sstevel@tonic-gate void 2537*0Sstevel@tonic-gate cpuset_only(cpuset_t *s, uint_t cpu) 2538*0Sstevel@tonic-gate { 2539*0Sstevel@tonic-gate CPUSET_ZERO(*s); 2540*0Sstevel@tonic-gate CPUSET_ADD(*s, cpu); 2541*0Sstevel@tonic-gate } 2542*0Sstevel@tonic-gate 2543*0Sstevel@tonic-gate int 2544*0Sstevel@tonic-gate cpuset_isnull(cpuset_t *s) 2545*0Sstevel@tonic-gate { 2546*0Sstevel@tonic-gate int i; 2547*0Sstevel@tonic-gate 2548*0Sstevel@tonic-gate for (i = 0; i < CPUSET_WORDS; i++) 2549*0Sstevel@tonic-gate if (s->cpub[i] != 0) 2550*0Sstevel@tonic-gate return (0); 2551*0Sstevel@tonic-gate return (1); 2552*0Sstevel@tonic-gate } 2553*0Sstevel@tonic-gate 2554*0Sstevel@tonic-gate int 2555*0Sstevel@tonic-gate cpuset_cmp(cpuset_t *s1, cpuset_t *s2) 2556*0Sstevel@tonic-gate { 2557*0Sstevel@tonic-gate int i; 2558*0Sstevel@tonic-gate 2559*0Sstevel@tonic-gate for (i = 0; i < CPUSET_WORDS; i++) 2560*0Sstevel@tonic-gate if (s1->cpub[i] != s2->cpub[i]) 2561*0Sstevel@tonic-gate return (0); 2562*0Sstevel@tonic-gate return (1); 2563*0Sstevel@tonic-gate } 2564*0Sstevel@tonic-gate 2565*0Sstevel@tonic-gate uint_t 2566*0Sstevel@tonic-gate cpuset_find(cpuset_t *s) 2567*0Sstevel@tonic-gate { 2568*0Sstevel@tonic-gate 2569*0Sstevel@tonic-gate uint_t i; 2570*0Sstevel@tonic-gate uint_t cpu = (uint_t)-1; 2571*0Sstevel@tonic-gate 2572*0Sstevel@tonic-gate /* 2573*0Sstevel@tonic-gate * Find a cpu in the cpuset 2574*0Sstevel@tonic-gate */ 2575*0Sstevel@tonic-gate for (i = 0; (i < CPUSET_WORDS && cpu == (uint_t)-1); i++) 2576*0Sstevel@tonic-gate cpu = (uint_t)(lowbit(s->cpub[i]) - 1); 2577*0Sstevel@tonic-gate return (cpu); 2578*0Sstevel@tonic-gate } 2579*0Sstevel@tonic-gate 2580*0Sstevel@tonic-gate #endif /* CPUSET_WORDS */ 2581*0Sstevel@tonic-gate 2582*0Sstevel@tonic-gate /* 2583*0Sstevel@tonic-gate * Unbind all user threads bound to a given CPU. 2584*0Sstevel@tonic-gate */ 2585*0Sstevel@tonic-gate int 2586*0Sstevel@tonic-gate cpu_unbind(processorid_t cpu) 2587*0Sstevel@tonic-gate { 2588*0Sstevel@tonic-gate processorid_t obind; 2589*0Sstevel@tonic-gate kthread_t *tp; 2590*0Sstevel@tonic-gate int ret = 0; 2591*0Sstevel@tonic-gate proc_t *pp; 2592*0Sstevel@tonic-gate int err, berr = 0; 2593*0Sstevel@tonic-gate 2594*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 2595*0Sstevel@tonic-gate 2596*0Sstevel@tonic-gate mutex_enter(&pidlock); 2597*0Sstevel@tonic-gate for (pp = practive; pp != NULL; pp = pp->p_next) { 2598*0Sstevel@tonic-gate mutex_enter(&pp->p_lock); 2599*0Sstevel@tonic-gate tp = pp->p_tlist; 2600*0Sstevel@tonic-gate /* 2601*0Sstevel@tonic-gate * Skip zombies, kernel processes, and processes in 2602*0Sstevel@tonic-gate * other zones, if called from a non-global zone. 2603*0Sstevel@tonic-gate */ 2604*0Sstevel@tonic-gate if (tp == NULL || (pp->p_flag & SSYS) || 2605*0Sstevel@tonic-gate !HASZONEACCESS(curproc, pp->p_zone->zone_id)) { 2606*0Sstevel@tonic-gate mutex_exit(&pp->p_lock); 2607*0Sstevel@tonic-gate continue; 2608*0Sstevel@tonic-gate } 2609*0Sstevel@tonic-gate do { 2610*0Sstevel@tonic-gate if (tp->t_bind_cpu != cpu) 2611*0Sstevel@tonic-gate continue; 2612*0Sstevel@tonic-gate err = cpu_bind_thread(tp, PBIND_NONE, &obind, &berr); 2613*0Sstevel@tonic-gate if (ret == 0) 2614*0Sstevel@tonic-gate ret = err; 2615*0Sstevel@tonic-gate } while ((tp = tp->t_forw) != pp->p_tlist); 2616*0Sstevel@tonic-gate mutex_exit(&pp->p_lock); 2617*0Sstevel@tonic-gate } 2618*0Sstevel@tonic-gate mutex_exit(&pidlock); 2619*0Sstevel@tonic-gate if (ret == 0) 2620*0Sstevel@tonic-gate ret = berr; 2621*0Sstevel@tonic-gate return (ret); 2622*0Sstevel@tonic-gate } 2623*0Sstevel@tonic-gate 2624*0Sstevel@tonic-gate 2625*0Sstevel@tonic-gate /* 2626*0Sstevel@tonic-gate * Destroy all remaining bound threads on a cpu. 2627*0Sstevel@tonic-gate */ 2628*0Sstevel@tonic-gate void 2629*0Sstevel@tonic-gate cpu_destroy_bound_threads(cpu_t *cp) 2630*0Sstevel@tonic-gate { 2631*0Sstevel@tonic-gate extern id_t syscid; 2632*0Sstevel@tonic-gate register kthread_id_t t, tlist, tnext; 2633*0Sstevel@tonic-gate 2634*0Sstevel@tonic-gate /* 2635*0Sstevel@tonic-gate * Destroy all remaining bound threads on the cpu. This 2636*0Sstevel@tonic-gate * should include both the interrupt threads and the idle thread. 2637*0Sstevel@tonic-gate * This requires some care, since we need to traverse the 2638*0Sstevel@tonic-gate * thread list with the pidlock mutex locked, but thread_free 2639*0Sstevel@tonic-gate * also locks the pidlock mutex. So, we collect the threads 2640*0Sstevel@tonic-gate * we're going to reap in a list headed by "tlist", then we 2641*0Sstevel@tonic-gate * unlock the pidlock mutex and traverse the tlist list, 2642*0Sstevel@tonic-gate * doing thread_free's on the thread's. Simple, n'est pas? 2643*0Sstevel@tonic-gate * Also, this depends on thread_free not mucking with the 2644*0Sstevel@tonic-gate * t_next and t_prev links of the thread. 2645*0Sstevel@tonic-gate */ 2646*0Sstevel@tonic-gate 2647*0Sstevel@tonic-gate if ((t = curthread) != NULL) { 2648*0Sstevel@tonic-gate 2649*0Sstevel@tonic-gate tlist = NULL; 2650*0Sstevel@tonic-gate mutex_enter(&pidlock); 2651*0Sstevel@tonic-gate do { 2652*0Sstevel@tonic-gate tnext = t->t_next; 2653*0Sstevel@tonic-gate if (t->t_bound_cpu == cp) { 2654*0Sstevel@tonic-gate 2655*0Sstevel@tonic-gate /* 2656*0Sstevel@tonic-gate * We've found a bound thread, carefully unlink 2657*0Sstevel@tonic-gate * it out of the thread list, and add it to 2658*0Sstevel@tonic-gate * our "tlist". We "know" we don't have to 2659*0Sstevel@tonic-gate * worry about unlinking curthread (the thread 2660*0Sstevel@tonic-gate * that is executing this code). 2661*0Sstevel@tonic-gate */ 2662*0Sstevel@tonic-gate t->t_next->t_prev = t->t_prev; 2663*0Sstevel@tonic-gate t->t_prev->t_next = t->t_next; 2664*0Sstevel@tonic-gate t->t_next = tlist; 2665*0Sstevel@tonic-gate tlist = t; 2666*0Sstevel@tonic-gate ASSERT(t->t_cid == syscid); 2667*0Sstevel@tonic-gate /* wake up anyone blocked in thread_join */ 2668*0Sstevel@tonic-gate cv_broadcast(&t->t_joincv); 2669*0Sstevel@tonic-gate /* 2670*0Sstevel@tonic-gate * t_lwp set by interrupt threads and not 2671*0Sstevel@tonic-gate * cleared. 2672*0Sstevel@tonic-gate */ 2673*0Sstevel@tonic-gate t->t_lwp = NULL; 2674*0Sstevel@tonic-gate /* 2675*0Sstevel@tonic-gate * Pause and idle threads always have 2676*0Sstevel@tonic-gate * t_state set to TS_ONPROC. 2677*0Sstevel@tonic-gate */ 2678*0Sstevel@tonic-gate t->t_state = TS_FREE; 2679*0Sstevel@tonic-gate t->t_prev = NULL; /* Just in case */ 2680*0Sstevel@tonic-gate } 2681*0Sstevel@tonic-gate 2682*0Sstevel@tonic-gate } while ((t = tnext) != curthread); 2683*0Sstevel@tonic-gate 2684*0Sstevel@tonic-gate mutex_exit(&pidlock); 2685*0Sstevel@tonic-gate 2686*0Sstevel@tonic-gate 2687*0Sstevel@tonic-gate for (t = tlist; t != NULL; t = tnext) { 2688*0Sstevel@tonic-gate tnext = t->t_next; 2689*0Sstevel@tonic-gate thread_free(t); 2690*0Sstevel@tonic-gate } 2691*0Sstevel@tonic-gate } 2692*0Sstevel@tonic-gate } 2693*0Sstevel@tonic-gate 2694*0Sstevel@tonic-gate /* 2695*0Sstevel@tonic-gate * processor_info(2) and p_online(2) status support functions 2696*0Sstevel@tonic-gate * The constants returned by the cpu_get_state() and cpu_get_state_str() are 2697*0Sstevel@tonic-gate * for use in communicating processor state information to userland. Kernel 2698*0Sstevel@tonic-gate * subsystems should only be using the cpu_flags value directly. Subsystems 2699*0Sstevel@tonic-gate * modifying cpu_flags should record the state change via a call to the 2700*0Sstevel@tonic-gate * cpu_set_state(). 2701*0Sstevel@tonic-gate */ 2702*0Sstevel@tonic-gate 2703*0Sstevel@tonic-gate /* 2704*0Sstevel@tonic-gate * Update the pi_state of this CPU. This function provides the CPU status for 2705*0Sstevel@tonic-gate * the information returned by processor_info(2). 2706*0Sstevel@tonic-gate */ 2707*0Sstevel@tonic-gate void 2708*0Sstevel@tonic-gate cpu_set_state(cpu_t *cpu) 2709*0Sstevel@tonic-gate { 2710*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 2711*0Sstevel@tonic-gate cpu->cpu_type_info.pi_state = cpu_get_state(cpu); 2712*0Sstevel@tonic-gate cpu->cpu_state_begin = gethrestime_sec(); 2713*0Sstevel@tonic-gate pool_cpu_mod = gethrtime(); 2714*0Sstevel@tonic-gate } 2715*0Sstevel@tonic-gate 2716*0Sstevel@tonic-gate /* 2717*0Sstevel@tonic-gate * Return offline/online/other status for the indicated CPU. Use only for 2718*0Sstevel@tonic-gate * communication with user applications; cpu_flags provides the in-kernel 2719*0Sstevel@tonic-gate * interface. 2720*0Sstevel@tonic-gate */ 2721*0Sstevel@tonic-gate int 2722*0Sstevel@tonic-gate cpu_get_state(cpu_t *cpu) 2723*0Sstevel@tonic-gate { 2724*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 2725*0Sstevel@tonic-gate if (cpu->cpu_flags & CPU_POWEROFF) 2726*0Sstevel@tonic-gate return (P_POWEROFF); 2727*0Sstevel@tonic-gate else if (cpu->cpu_flags & CPU_FAULTED) 2728*0Sstevel@tonic-gate return (P_FAULTED); 2729*0Sstevel@tonic-gate else if (cpu->cpu_flags & CPU_SPARE) 2730*0Sstevel@tonic-gate return (P_SPARE); 2731*0Sstevel@tonic-gate else if ((cpu->cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY) 2732*0Sstevel@tonic-gate return (P_OFFLINE); 2733*0Sstevel@tonic-gate else if (cpu->cpu_flags & CPU_ENABLE) 2734*0Sstevel@tonic-gate return (P_ONLINE); 2735*0Sstevel@tonic-gate else 2736*0Sstevel@tonic-gate return (P_NOINTR); 2737*0Sstevel@tonic-gate } 2738*0Sstevel@tonic-gate 2739*0Sstevel@tonic-gate /* 2740*0Sstevel@tonic-gate * Return processor_info(2) state as a string. 2741*0Sstevel@tonic-gate */ 2742*0Sstevel@tonic-gate const char * 2743*0Sstevel@tonic-gate cpu_get_state_str(cpu_t *cpu) 2744*0Sstevel@tonic-gate { 2745*0Sstevel@tonic-gate const char *string; 2746*0Sstevel@tonic-gate 2747*0Sstevel@tonic-gate switch (cpu_get_state(cpu)) { 2748*0Sstevel@tonic-gate case P_ONLINE: 2749*0Sstevel@tonic-gate string = PS_ONLINE; 2750*0Sstevel@tonic-gate break; 2751*0Sstevel@tonic-gate case P_POWEROFF: 2752*0Sstevel@tonic-gate string = PS_POWEROFF; 2753*0Sstevel@tonic-gate break; 2754*0Sstevel@tonic-gate case P_NOINTR: 2755*0Sstevel@tonic-gate string = PS_NOINTR; 2756*0Sstevel@tonic-gate break; 2757*0Sstevel@tonic-gate case P_SPARE: 2758*0Sstevel@tonic-gate string = PS_SPARE; 2759*0Sstevel@tonic-gate break; 2760*0Sstevel@tonic-gate case P_FAULTED: 2761*0Sstevel@tonic-gate string = PS_FAULTED; 2762*0Sstevel@tonic-gate break; 2763*0Sstevel@tonic-gate case P_OFFLINE: 2764*0Sstevel@tonic-gate string = PS_OFFLINE; 2765*0Sstevel@tonic-gate break; 2766*0Sstevel@tonic-gate default: 2767*0Sstevel@tonic-gate string = "unknown"; 2768*0Sstevel@tonic-gate break; 2769*0Sstevel@tonic-gate } 2770*0Sstevel@tonic-gate return (string); 2771*0Sstevel@tonic-gate } 2772*0Sstevel@tonic-gate 2773*0Sstevel@tonic-gate /* 2774*0Sstevel@tonic-gate * Export this CPU's statistics (cpu_stat_t and cpu_stats_t) as raw and named 2775*0Sstevel@tonic-gate * kstats, respectively. This is done when a CPU is initialized or placed 2776*0Sstevel@tonic-gate * online via p_online(2). 2777*0Sstevel@tonic-gate */ 2778*0Sstevel@tonic-gate static void 2779*0Sstevel@tonic-gate cpu_stats_kstat_create(cpu_t *cp) 2780*0Sstevel@tonic-gate { 2781*0Sstevel@tonic-gate int instance = cp->cpu_id; 2782*0Sstevel@tonic-gate char *module = "cpu"; 2783*0Sstevel@tonic-gate char *class = "misc"; 2784*0Sstevel@tonic-gate kstat_t *ksp; 2785*0Sstevel@tonic-gate zoneid_t zoneid; 2786*0Sstevel@tonic-gate 2787*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 2788*0Sstevel@tonic-gate 2789*0Sstevel@tonic-gate if (pool_pset_enabled()) 2790*0Sstevel@tonic-gate zoneid = GLOBAL_ZONEID; 2791*0Sstevel@tonic-gate else 2792*0Sstevel@tonic-gate zoneid = ALL_ZONES; 2793*0Sstevel@tonic-gate /* 2794*0Sstevel@tonic-gate * Create named kstats 2795*0Sstevel@tonic-gate */ 2796*0Sstevel@tonic-gate #define CPU_STATS_KS_CREATE(name, tsize, update_func) \ 2797*0Sstevel@tonic-gate ksp = kstat_create_zone(module, instance, (name), class, \ 2798*0Sstevel@tonic-gate KSTAT_TYPE_NAMED, (tsize) / sizeof (kstat_named_t), 0, \ 2799*0Sstevel@tonic-gate zoneid); \ 2800*0Sstevel@tonic-gate if (ksp != NULL) { \ 2801*0Sstevel@tonic-gate ksp->ks_private = cp; \ 2802*0Sstevel@tonic-gate ksp->ks_update = (update_func); \ 2803*0Sstevel@tonic-gate kstat_install(ksp); \ 2804*0Sstevel@tonic-gate } else \ 2805*0Sstevel@tonic-gate cmn_err(CE_WARN, "cpu: unable to create %s:%d:%s kstat", \ 2806*0Sstevel@tonic-gate module, instance, (name)); 2807*0Sstevel@tonic-gate 2808*0Sstevel@tonic-gate CPU_STATS_KS_CREATE("sys", sizeof (cpu_sys_stats_ks_data_template), 2809*0Sstevel@tonic-gate cpu_sys_stats_ks_update); 2810*0Sstevel@tonic-gate CPU_STATS_KS_CREATE("vm", sizeof (cpu_vm_stats_ks_data_template), 2811*0Sstevel@tonic-gate cpu_vm_stats_ks_update); 2812*0Sstevel@tonic-gate 2813*0Sstevel@tonic-gate /* 2814*0Sstevel@tonic-gate * Export the familiar cpu_stat_t KSTAT_TYPE_RAW kstat. 2815*0Sstevel@tonic-gate */ 2816*0Sstevel@tonic-gate ksp = kstat_create_zone("cpu_stat", cp->cpu_id, NULL, 2817*0Sstevel@tonic-gate "misc", KSTAT_TYPE_RAW, sizeof (cpu_stat_t), 0, zoneid); 2818*0Sstevel@tonic-gate if (ksp != NULL) { 2819*0Sstevel@tonic-gate ksp->ks_update = cpu_stat_ks_update; 2820*0Sstevel@tonic-gate ksp->ks_private = cp; 2821*0Sstevel@tonic-gate kstat_install(ksp); 2822*0Sstevel@tonic-gate } 2823*0Sstevel@tonic-gate } 2824*0Sstevel@tonic-gate 2825*0Sstevel@tonic-gate static void 2826*0Sstevel@tonic-gate cpu_stats_kstat_destroy(cpu_t *cp) 2827*0Sstevel@tonic-gate { 2828*0Sstevel@tonic-gate char ks_name[KSTAT_STRLEN]; 2829*0Sstevel@tonic-gate 2830*0Sstevel@tonic-gate (void) sprintf(ks_name, "cpu_stat%d", cp->cpu_id); 2831*0Sstevel@tonic-gate kstat_delete_byname("cpu_stat", cp->cpu_id, ks_name); 2832*0Sstevel@tonic-gate 2833*0Sstevel@tonic-gate kstat_delete_byname("cpu", cp->cpu_id, "sys"); 2834*0Sstevel@tonic-gate kstat_delete_byname("cpu", cp->cpu_id, "vm"); 2835*0Sstevel@tonic-gate } 2836*0Sstevel@tonic-gate 2837*0Sstevel@tonic-gate static int 2838*0Sstevel@tonic-gate cpu_sys_stats_ks_update(kstat_t *ksp, int rw) 2839*0Sstevel@tonic-gate { 2840*0Sstevel@tonic-gate cpu_t *cp = (cpu_t *)ksp->ks_private; 2841*0Sstevel@tonic-gate struct cpu_sys_stats_ks_data *csskd; 2842*0Sstevel@tonic-gate cpu_sys_stats_t *css; 2843*0Sstevel@tonic-gate hrtime_t *cmstimep; 2844*0Sstevel@tonic-gate hrtime_t newtime; 2845*0Sstevel@tonic-gate int i; 2846*0Sstevel@tonic-gate 2847*0Sstevel@tonic-gate if (rw == KSTAT_WRITE) 2848*0Sstevel@tonic-gate return (EACCES); 2849*0Sstevel@tonic-gate 2850*0Sstevel@tonic-gate csskd = ksp->ks_data; 2851*0Sstevel@tonic-gate css = &cp->cpu_stats.sys; 2852*0Sstevel@tonic-gate 2853*0Sstevel@tonic-gate bcopy(&cpu_sys_stats_ks_data_template, ksp->ks_data, 2854*0Sstevel@tonic-gate sizeof (cpu_sys_stats_ks_data_template)); 2855*0Sstevel@tonic-gate csskd->cpu_ticks_wait.value.ui64 = 0; 2856*0Sstevel@tonic-gate csskd->wait_ticks_io.value.ui64 = 0; 2857*0Sstevel@tonic-gate csskd->cpu_nsec_idle.value.ui64 = cp->cpu_acct[CMS_IDLE]; 2858*0Sstevel@tonic-gate csskd->cpu_nsec_user.value.ui64 = cp->cpu_acct[CMS_USER]; 2859*0Sstevel@tonic-gate csskd->cpu_nsec_kernel.value.ui64 = cp->cpu_acct[CMS_SYSTEM]; 2860*0Sstevel@tonic-gate 2861*0Sstevel@tonic-gate /* 2862*0Sstevel@tonic-gate * update kstats to include time spent in current state 2863*0Sstevel@tonic-gate */ 2864*0Sstevel@tonic-gate i = 0; /* set counter to 0 */ 2865*0Sstevel@tonic-gate do { 2866*0Sstevel@tonic-gate switch (cp->cpu_mstate) { 2867*0Sstevel@tonic-gate case CMS_USER: 2868*0Sstevel@tonic-gate cmstimep = (hrtime_t *)&csskd->cpu_nsec_user.value.ui64; 2869*0Sstevel@tonic-gate break; 2870*0Sstevel@tonic-gate case CMS_SYSTEM: 2871*0Sstevel@tonic-gate cmstimep = 2872*0Sstevel@tonic-gate (hrtime_t *)&csskd->cpu_nsec_kernel.value.ui64; 2873*0Sstevel@tonic-gate break; 2874*0Sstevel@tonic-gate case CMS_IDLE: 2875*0Sstevel@tonic-gate cmstimep = (hrtime_t *)&csskd->cpu_nsec_idle.value.ui64; 2876*0Sstevel@tonic-gate break; 2877*0Sstevel@tonic-gate case CMS_DISABLED: 2878*0Sstevel@tonic-gate panic("cpu_sys_stats_ks_update: disabled state" 2879*0Sstevel@tonic-gate " reported!"); 2880*0Sstevel@tonic-gate break; 2881*0Sstevel@tonic-gate default: 2882*0Sstevel@tonic-gate panic("cpu_sys_stats_ks_update: unknown microstate!"); 2883*0Sstevel@tonic-gate } 2884*0Sstevel@tonic-gate newtime = gethrtime_unscaled() - cp->cpu_mstate_start; 2885*0Sstevel@tonic-gate if (newtime < 0) { 2886*0Sstevel@tonic-gate newtime = 0; 2887*0Sstevel@tonic-gate i++; 2888*0Sstevel@tonic-gate } 2889*0Sstevel@tonic-gate } while (newtime <= 0 && i < 5); 2890*0Sstevel@tonic-gate 2891*0Sstevel@tonic-gate *cmstimep += newtime; 2892*0Sstevel@tonic-gate 2893*0Sstevel@tonic-gate scalehrtime((hrtime_t *)&csskd->cpu_nsec_idle.value.ui64); 2894*0Sstevel@tonic-gate scalehrtime((hrtime_t *)&csskd->cpu_nsec_user.value.ui64); 2895*0Sstevel@tonic-gate scalehrtime((hrtime_t *)&csskd->cpu_nsec_kernel.value.ui64); 2896*0Sstevel@tonic-gate csskd->cpu_ticks_idle.value.ui64 = 2897*0Sstevel@tonic-gate NSEC_TO_TICK(csskd->cpu_nsec_idle.value.ui64); 2898*0Sstevel@tonic-gate csskd->cpu_ticks_user.value.ui64 = 2899*0Sstevel@tonic-gate NSEC_TO_TICK(csskd->cpu_nsec_user.value.ui64); 2900*0Sstevel@tonic-gate csskd->cpu_ticks_kernel.value.ui64 = 2901*0Sstevel@tonic-gate NSEC_TO_TICK(csskd->cpu_nsec_kernel.value.ui64); 2902*0Sstevel@tonic-gate csskd->bread.value.ui64 = css->bread; 2903*0Sstevel@tonic-gate csskd->bwrite.value.ui64 = css->bwrite; 2904*0Sstevel@tonic-gate csskd->lread.value.ui64 = css->lread; 2905*0Sstevel@tonic-gate csskd->lwrite.value.ui64 = css->lwrite; 2906*0Sstevel@tonic-gate csskd->phread.value.ui64 = css->phread; 2907*0Sstevel@tonic-gate csskd->phwrite.value.ui64 = css->phwrite; 2908*0Sstevel@tonic-gate csskd->pswitch.value.ui64 = css->pswitch; 2909*0Sstevel@tonic-gate csskd->trap.value.ui64 = css->trap; 2910*0Sstevel@tonic-gate csskd->intr.value.ui64 = 0; 2911*0Sstevel@tonic-gate for (i = 0; i < PIL_MAX; i++) 2912*0Sstevel@tonic-gate csskd->intr.value.ui64 += css->intr[i]; 2913*0Sstevel@tonic-gate csskd->syscall.value.ui64 = css->syscall; 2914*0Sstevel@tonic-gate csskd->sysread.value.ui64 = css->sysread; 2915*0Sstevel@tonic-gate csskd->syswrite.value.ui64 = css->syswrite; 2916*0Sstevel@tonic-gate csskd->sysfork.value.ui64 = css->sysfork; 2917*0Sstevel@tonic-gate csskd->sysvfork.value.ui64 = css->sysvfork; 2918*0Sstevel@tonic-gate csskd->sysexec.value.ui64 = css->sysexec; 2919*0Sstevel@tonic-gate csskd->readch.value.ui64 = css->readch; 2920*0Sstevel@tonic-gate csskd->writech.value.ui64 = css->writech; 2921*0Sstevel@tonic-gate csskd->rcvint.value.ui64 = css->rcvint; 2922*0Sstevel@tonic-gate csskd->xmtint.value.ui64 = css->xmtint; 2923*0Sstevel@tonic-gate csskd->mdmint.value.ui64 = css->mdmint; 2924*0Sstevel@tonic-gate csskd->rawch.value.ui64 = css->rawch; 2925*0Sstevel@tonic-gate csskd->canch.value.ui64 = css->canch; 2926*0Sstevel@tonic-gate csskd->outch.value.ui64 = css->outch; 2927*0Sstevel@tonic-gate csskd->msg.value.ui64 = css->msg; 2928*0Sstevel@tonic-gate csskd->sema.value.ui64 = css->sema; 2929*0Sstevel@tonic-gate csskd->namei.value.ui64 = css->namei; 2930*0Sstevel@tonic-gate csskd->ufsiget.value.ui64 = css->ufsiget; 2931*0Sstevel@tonic-gate csskd->ufsdirblk.value.ui64 = css->ufsdirblk; 2932*0Sstevel@tonic-gate csskd->ufsipage.value.ui64 = css->ufsipage; 2933*0Sstevel@tonic-gate csskd->ufsinopage.value.ui64 = css->ufsinopage; 2934*0Sstevel@tonic-gate csskd->procovf.value.ui64 = css->procovf; 2935*0Sstevel@tonic-gate csskd->intrthread.value.ui64 = 0; 2936*0Sstevel@tonic-gate for (i = 0; i < LOCK_LEVEL; i++) 2937*0Sstevel@tonic-gate csskd->intrthread.value.ui64 += css->intr[i]; 2938*0Sstevel@tonic-gate csskd->intrblk.value.ui64 = css->intrblk; 2939*0Sstevel@tonic-gate csskd->intrunpin.value.ui64 = css->intrunpin; 2940*0Sstevel@tonic-gate csskd->idlethread.value.ui64 = css->idlethread; 2941*0Sstevel@tonic-gate csskd->inv_swtch.value.ui64 = css->inv_swtch; 2942*0Sstevel@tonic-gate csskd->nthreads.value.ui64 = css->nthreads; 2943*0Sstevel@tonic-gate csskd->cpumigrate.value.ui64 = css->cpumigrate; 2944*0Sstevel@tonic-gate csskd->xcalls.value.ui64 = css->xcalls; 2945*0Sstevel@tonic-gate csskd->mutex_adenters.value.ui64 = css->mutex_adenters; 2946*0Sstevel@tonic-gate csskd->rw_rdfails.value.ui64 = css->rw_rdfails; 2947*0Sstevel@tonic-gate csskd->rw_wrfails.value.ui64 = css->rw_wrfails; 2948*0Sstevel@tonic-gate csskd->modload.value.ui64 = css->modload; 2949*0Sstevel@tonic-gate csskd->modunload.value.ui64 = css->modunload; 2950*0Sstevel@tonic-gate csskd->bawrite.value.ui64 = css->bawrite; 2951*0Sstevel@tonic-gate csskd->iowait.value.ui64 = 0; 2952*0Sstevel@tonic-gate 2953*0Sstevel@tonic-gate return (0); 2954*0Sstevel@tonic-gate } 2955*0Sstevel@tonic-gate 2956*0Sstevel@tonic-gate static int 2957*0Sstevel@tonic-gate cpu_vm_stats_ks_update(kstat_t *ksp, int rw) 2958*0Sstevel@tonic-gate { 2959*0Sstevel@tonic-gate cpu_t *cp = (cpu_t *)ksp->ks_private; 2960*0Sstevel@tonic-gate struct cpu_vm_stats_ks_data *cvskd; 2961*0Sstevel@tonic-gate cpu_vm_stats_t *cvs; 2962*0Sstevel@tonic-gate 2963*0Sstevel@tonic-gate if (rw == KSTAT_WRITE) 2964*0Sstevel@tonic-gate return (EACCES); 2965*0Sstevel@tonic-gate 2966*0Sstevel@tonic-gate cvs = &cp->cpu_stats.vm; 2967*0Sstevel@tonic-gate cvskd = ksp->ks_data; 2968*0Sstevel@tonic-gate 2969*0Sstevel@tonic-gate bcopy(&cpu_vm_stats_ks_data_template, ksp->ks_data, 2970*0Sstevel@tonic-gate sizeof (cpu_vm_stats_ks_data_template)); 2971*0Sstevel@tonic-gate cvskd->pgrec.value.ui64 = cvs->pgrec; 2972*0Sstevel@tonic-gate cvskd->pgfrec.value.ui64 = cvs->pgfrec; 2973*0Sstevel@tonic-gate cvskd->pgin.value.ui64 = cvs->pgin; 2974*0Sstevel@tonic-gate cvskd->pgpgin.value.ui64 = cvs->pgpgin; 2975*0Sstevel@tonic-gate cvskd->pgout.value.ui64 = cvs->pgout; 2976*0Sstevel@tonic-gate cvskd->pgpgout.value.ui64 = cvs->pgpgout; 2977*0Sstevel@tonic-gate cvskd->swapin.value.ui64 = cvs->swapin; 2978*0Sstevel@tonic-gate cvskd->pgswapin.value.ui64 = cvs->pgswapin; 2979*0Sstevel@tonic-gate cvskd->swapout.value.ui64 = cvs->swapout; 2980*0Sstevel@tonic-gate cvskd->pgswapout.value.ui64 = cvs->pgswapout; 2981*0Sstevel@tonic-gate cvskd->zfod.value.ui64 = cvs->zfod; 2982*0Sstevel@tonic-gate cvskd->dfree.value.ui64 = cvs->dfree; 2983*0Sstevel@tonic-gate cvskd->scan.value.ui64 = cvs->scan; 2984*0Sstevel@tonic-gate cvskd->rev.value.ui64 = cvs->rev; 2985*0Sstevel@tonic-gate cvskd->hat_fault.value.ui64 = cvs->hat_fault; 2986*0Sstevel@tonic-gate cvskd->as_fault.value.ui64 = cvs->as_fault; 2987*0Sstevel@tonic-gate cvskd->maj_fault.value.ui64 = cvs->maj_fault; 2988*0Sstevel@tonic-gate cvskd->cow_fault.value.ui64 = cvs->cow_fault; 2989*0Sstevel@tonic-gate cvskd->prot_fault.value.ui64 = cvs->prot_fault; 2990*0Sstevel@tonic-gate cvskd->softlock.value.ui64 = cvs->softlock; 2991*0Sstevel@tonic-gate cvskd->kernel_asflt.value.ui64 = cvs->kernel_asflt; 2992*0Sstevel@tonic-gate cvskd->pgrrun.value.ui64 = cvs->pgrrun; 2993*0Sstevel@tonic-gate cvskd->execpgin.value.ui64 = cvs->execpgin; 2994*0Sstevel@tonic-gate cvskd->execpgout.value.ui64 = cvs->execpgout; 2995*0Sstevel@tonic-gate cvskd->execfree.value.ui64 = cvs->execfree; 2996*0Sstevel@tonic-gate cvskd->anonpgin.value.ui64 = cvs->anonpgin; 2997*0Sstevel@tonic-gate cvskd->anonpgout.value.ui64 = cvs->anonpgout; 2998*0Sstevel@tonic-gate cvskd->anonfree.value.ui64 = cvs->anonfree; 2999*0Sstevel@tonic-gate cvskd->fspgin.value.ui64 = cvs->fspgin; 3000*0Sstevel@tonic-gate cvskd->fspgout.value.ui64 = cvs->fspgout; 3001*0Sstevel@tonic-gate cvskd->fsfree.value.ui64 = cvs->fsfree; 3002*0Sstevel@tonic-gate 3003*0Sstevel@tonic-gate return (0); 3004*0Sstevel@tonic-gate } 3005*0Sstevel@tonic-gate 3006*0Sstevel@tonic-gate static int 3007*0Sstevel@tonic-gate cpu_stat_ks_update(kstat_t *ksp, int rw) 3008*0Sstevel@tonic-gate { 3009*0Sstevel@tonic-gate cpu_stat_t *cso; 3010*0Sstevel@tonic-gate cpu_t *cp; 3011*0Sstevel@tonic-gate int i; 3012*0Sstevel@tonic-gate hrtime_t newtime; 3013*0Sstevel@tonic-gate hrtime_t cphrt_i; 3014*0Sstevel@tonic-gate hrtime_t cphrt_u; 3015*0Sstevel@tonic-gate hrtime_t cphrt_s; 3016*0Sstevel@tonic-gate hrtime_t *cmstimep; 3017*0Sstevel@tonic-gate 3018*0Sstevel@tonic-gate cso = (cpu_stat_t *)ksp->ks_data; 3019*0Sstevel@tonic-gate cp = (cpu_t *)ksp->ks_private; 3020*0Sstevel@tonic-gate 3021*0Sstevel@tonic-gate if (rw == KSTAT_WRITE) 3022*0Sstevel@tonic-gate return (EACCES); 3023*0Sstevel@tonic-gate 3024*0Sstevel@tonic-gate cphrt_i = cp->cpu_acct[CMS_IDLE]; 3025*0Sstevel@tonic-gate cphrt_u = cp->cpu_acct[CMS_USER]; 3026*0Sstevel@tonic-gate cphrt_s = cp->cpu_acct[CMS_SYSTEM]; 3027*0Sstevel@tonic-gate 3028*0Sstevel@tonic-gate /* 3029*0Sstevel@tonic-gate * update kstats to include time spent in current state 3030*0Sstevel@tonic-gate */ 3031*0Sstevel@tonic-gate i = 0; /* set counter to 0 */ 3032*0Sstevel@tonic-gate do { 3033*0Sstevel@tonic-gate switch (cp->cpu_mstate) { 3034*0Sstevel@tonic-gate case CMS_USER: 3035*0Sstevel@tonic-gate cmstimep = &cphrt_u; 3036*0Sstevel@tonic-gate break; 3037*0Sstevel@tonic-gate case CMS_SYSTEM: 3038*0Sstevel@tonic-gate cmstimep = &cphrt_s; 3039*0Sstevel@tonic-gate break; 3040*0Sstevel@tonic-gate case CMS_IDLE: 3041*0Sstevel@tonic-gate cmstimep = &cphrt_i; 3042*0Sstevel@tonic-gate break; 3043*0Sstevel@tonic-gate case CMS_DISABLED: 3044*0Sstevel@tonic-gate panic("cpu_stat_ks_update: disabled state reported!"); 3045*0Sstevel@tonic-gate break; 3046*0Sstevel@tonic-gate default: 3047*0Sstevel@tonic-gate panic("cpu_stat_ks_update: unknown microstate!"); 3048*0Sstevel@tonic-gate } 3049*0Sstevel@tonic-gate newtime = gethrtime_unscaled() - cp->cpu_mstate_start; 3050*0Sstevel@tonic-gate if (newtime < 0) { 3051*0Sstevel@tonic-gate newtime = 0; 3052*0Sstevel@tonic-gate i++; 3053*0Sstevel@tonic-gate } 3054*0Sstevel@tonic-gate } while (newtime <= 0 && i < 5); 3055*0Sstevel@tonic-gate 3056*0Sstevel@tonic-gate *cmstimep += newtime; 3057*0Sstevel@tonic-gate 3058*0Sstevel@tonic-gate scalehrtime((hrtime_t *)&cphrt_i); 3059*0Sstevel@tonic-gate cso->cpu_sysinfo.cpu[CPU_IDLE] = NSEC_TO_TICK(cphrt_i); 3060*0Sstevel@tonic-gate 3061*0Sstevel@tonic-gate scalehrtime((hrtime_t *)&cphrt_u); 3062*0Sstevel@tonic-gate cso->cpu_sysinfo.cpu[CPU_USER] = NSEC_TO_TICK(cphrt_u); 3063*0Sstevel@tonic-gate 3064*0Sstevel@tonic-gate scalehrtime((hrtime_t *)&cphrt_s); 3065*0Sstevel@tonic-gate cso->cpu_sysinfo.cpu[CPU_KERNEL] = NSEC_TO_TICK(cphrt_s); 3066*0Sstevel@tonic-gate 3067*0Sstevel@tonic-gate cso->cpu_sysinfo.cpu[CPU_WAIT] = 0; 3068*0Sstevel@tonic-gate cso->cpu_sysinfo.wait[W_IO] = 0; 3069*0Sstevel@tonic-gate cso->cpu_sysinfo.wait[W_SWAP] = 0; 3070*0Sstevel@tonic-gate cso->cpu_sysinfo.wait[W_PIO] = 0; 3071*0Sstevel@tonic-gate cso->cpu_sysinfo.bread = CPU_STATS(cp, sys.bread); 3072*0Sstevel@tonic-gate cso->cpu_sysinfo.bwrite = CPU_STATS(cp, sys.bwrite); 3073*0Sstevel@tonic-gate cso->cpu_sysinfo.lread = CPU_STATS(cp, sys.lread); 3074*0Sstevel@tonic-gate cso->cpu_sysinfo.lwrite = CPU_STATS(cp, sys.lwrite); 3075*0Sstevel@tonic-gate cso->cpu_sysinfo.phread = CPU_STATS(cp, sys.phread); 3076*0Sstevel@tonic-gate cso->cpu_sysinfo.phwrite = CPU_STATS(cp, sys.phwrite); 3077*0Sstevel@tonic-gate cso->cpu_sysinfo.pswitch = CPU_STATS(cp, sys.pswitch); 3078*0Sstevel@tonic-gate cso->cpu_sysinfo.trap = CPU_STATS(cp, sys.trap); 3079*0Sstevel@tonic-gate cso->cpu_sysinfo.intr = 0; 3080*0Sstevel@tonic-gate for (i = 0; i < PIL_MAX; i++) 3081*0Sstevel@tonic-gate cso->cpu_sysinfo.intr += CPU_STATS(cp, sys.intr[i]); 3082*0Sstevel@tonic-gate cso->cpu_sysinfo.syscall = CPU_STATS(cp, sys.syscall); 3083*0Sstevel@tonic-gate cso->cpu_sysinfo.sysread = CPU_STATS(cp, sys.sysread); 3084*0Sstevel@tonic-gate cso->cpu_sysinfo.syswrite = CPU_STATS(cp, sys.syswrite); 3085*0Sstevel@tonic-gate cso->cpu_sysinfo.sysfork = CPU_STATS(cp, sys.sysfork); 3086*0Sstevel@tonic-gate cso->cpu_sysinfo.sysvfork = CPU_STATS(cp, sys.sysvfork); 3087*0Sstevel@tonic-gate cso->cpu_sysinfo.sysexec = CPU_STATS(cp, sys.sysexec); 3088*0Sstevel@tonic-gate cso->cpu_sysinfo.readch = CPU_STATS(cp, sys.readch); 3089*0Sstevel@tonic-gate cso->cpu_sysinfo.writech = CPU_STATS(cp, sys.writech); 3090*0Sstevel@tonic-gate cso->cpu_sysinfo.rcvint = CPU_STATS(cp, sys.rcvint); 3091*0Sstevel@tonic-gate cso->cpu_sysinfo.xmtint = CPU_STATS(cp, sys.xmtint); 3092*0Sstevel@tonic-gate cso->cpu_sysinfo.mdmint = CPU_STATS(cp, sys.mdmint); 3093*0Sstevel@tonic-gate cso->cpu_sysinfo.rawch = CPU_STATS(cp, sys.rawch); 3094*0Sstevel@tonic-gate cso->cpu_sysinfo.canch = CPU_STATS(cp, sys.canch); 3095*0Sstevel@tonic-gate cso->cpu_sysinfo.outch = CPU_STATS(cp, sys.outch); 3096*0Sstevel@tonic-gate cso->cpu_sysinfo.msg = CPU_STATS(cp, sys.msg); 3097*0Sstevel@tonic-gate cso->cpu_sysinfo.sema = CPU_STATS(cp, sys.sema); 3098*0Sstevel@tonic-gate cso->cpu_sysinfo.namei = CPU_STATS(cp, sys.namei); 3099*0Sstevel@tonic-gate cso->cpu_sysinfo.ufsiget = CPU_STATS(cp, sys.ufsiget); 3100*0Sstevel@tonic-gate cso->cpu_sysinfo.ufsdirblk = CPU_STATS(cp, sys.ufsdirblk); 3101*0Sstevel@tonic-gate cso->cpu_sysinfo.ufsipage = CPU_STATS(cp, sys.ufsipage); 3102*0Sstevel@tonic-gate cso->cpu_sysinfo.ufsinopage = CPU_STATS(cp, sys.ufsinopage); 3103*0Sstevel@tonic-gate cso->cpu_sysinfo.inodeovf = 0; 3104*0Sstevel@tonic-gate cso->cpu_sysinfo.fileovf = 0; 3105*0Sstevel@tonic-gate cso->cpu_sysinfo.procovf = CPU_STATS(cp, sys.procovf); 3106*0Sstevel@tonic-gate cso->cpu_sysinfo.intrthread = 0; 3107*0Sstevel@tonic-gate for (i = 0; i < LOCK_LEVEL; i++) 3108*0Sstevel@tonic-gate cso->cpu_sysinfo.intrthread += CPU_STATS(cp, sys.intr[i]); 3109*0Sstevel@tonic-gate cso->cpu_sysinfo.intrblk = CPU_STATS(cp, sys.intrblk); 3110*0Sstevel@tonic-gate cso->cpu_sysinfo.idlethread = CPU_STATS(cp, sys.idlethread); 3111*0Sstevel@tonic-gate cso->cpu_sysinfo.inv_swtch = CPU_STATS(cp, sys.inv_swtch); 3112*0Sstevel@tonic-gate cso->cpu_sysinfo.nthreads = CPU_STATS(cp, sys.nthreads); 3113*0Sstevel@tonic-gate cso->cpu_sysinfo.cpumigrate = CPU_STATS(cp, sys.cpumigrate); 3114*0Sstevel@tonic-gate cso->cpu_sysinfo.xcalls = CPU_STATS(cp, sys.xcalls); 3115*0Sstevel@tonic-gate cso->cpu_sysinfo.mutex_adenters = CPU_STATS(cp, sys.mutex_adenters); 3116*0Sstevel@tonic-gate cso->cpu_sysinfo.rw_rdfails = CPU_STATS(cp, sys.rw_rdfails); 3117*0Sstevel@tonic-gate cso->cpu_sysinfo.rw_wrfails = CPU_STATS(cp, sys.rw_wrfails); 3118*0Sstevel@tonic-gate cso->cpu_sysinfo.modload = CPU_STATS(cp, sys.modload); 3119*0Sstevel@tonic-gate cso->cpu_sysinfo.modunload = CPU_STATS(cp, sys.modunload); 3120*0Sstevel@tonic-gate cso->cpu_sysinfo.bawrite = CPU_STATS(cp, sys.bawrite); 3121*0Sstevel@tonic-gate cso->cpu_sysinfo.rw_enters = 0; 3122*0Sstevel@tonic-gate cso->cpu_sysinfo.win_uo_cnt = 0; 3123*0Sstevel@tonic-gate cso->cpu_sysinfo.win_uu_cnt = 0; 3124*0Sstevel@tonic-gate cso->cpu_sysinfo.win_so_cnt = 0; 3125*0Sstevel@tonic-gate cso->cpu_sysinfo.win_su_cnt = 0; 3126*0Sstevel@tonic-gate cso->cpu_sysinfo.win_suo_cnt = 0; 3127*0Sstevel@tonic-gate 3128*0Sstevel@tonic-gate cso->cpu_syswait.iowait = 0; 3129*0Sstevel@tonic-gate cso->cpu_syswait.swap = 0; 3130*0Sstevel@tonic-gate cso->cpu_syswait.physio = 0; 3131*0Sstevel@tonic-gate 3132*0Sstevel@tonic-gate cso->cpu_vminfo.pgrec = CPU_STATS(cp, vm.pgrec); 3133*0Sstevel@tonic-gate cso->cpu_vminfo.pgfrec = CPU_STATS(cp, vm.pgfrec); 3134*0Sstevel@tonic-gate cso->cpu_vminfo.pgin = CPU_STATS(cp, vm.pgin); 3135*0Sstevel@tonic-gate cso->cpu_vminfo.pgpgin = CPU_STATS(cp, vm.pgpgin); 3136*0Sstevel@tonic-gate cso->cpu_vminfo.pgout = CPU_STATS(cp, vm.pgout); 3137*0Sstevel@tonic-gate cso->cpu_vminfo.pgpgout = CPU_STATS(cp, vm.pgpgout); 3138*0Sstevel@tonic-gate cso->cpu_vminfo.swapin = CPU_STATS(cp, vm.swapin); 3139*0Sstevel@tonic-gate cso->cpu_vminfo.pgswapin = CPU_STATS(cp, vm.pgswapin); 3140*0Sstevel@tonic-gate cso->cpu_vminfo.swapout = CPU_STATS(cp, vm.swapout); 3141*0Sstevel@tonic-gate cso->cpu_vminfo.pgswapout = CPU_STATS(cp, vm.pgswapout); 3142*0Sstevel@tonic-gate cso->cpu_vminfo.zfod = CPU_STATS(cp, vm.zfod); 3143*0Sstevel@tonic-gate cso->cpu_vminfo.dfree = CPU_STATS(cp, vm.dfree); 3144*0Sstevel@tonic-gate cso->cpu_vminfo.scan = CPU_STATS(cp, vm.scan); 3145*0Sstevel@tonic-gate cso->cpu_vminfo.rev = CPU_STATS(cp, vm.rev); 3146*0Sstevel@tonic-gate cso->cpu_vminfo.hat_fault = CPU_STATS(cp, vm.hat_fault); 3147*0Sstevel@tonic-gate cso->cpu_vminfo.as_fault = CPU_STATS(cp, vm.as_fault); 3148*0Sstevel@tonic-gate cso->cpu_vminfo.maj_fault = CPU_STATS(cp, vm.maj_fault); 3149*0Sstevel@tonic-gate cso->cpu_vminfo.cow_fault = CPU_STATS(cp, vm.cow_fault); 3150*0Sstevel@tonic-gate cso->cpu_vminfo.prot_fault = CPU_STATS(cp, vm.prot_fault); 3151*0Sstevel@tonic-gate cso->cpu_vminfo.softlock = CPU_STATS(cp, vm.softlock); 3152*0Sstevel@tonic-gate cso->cpu_vminfo.kernel_asflt = CPU_STATS(cp, vm.kernel_asflt); 3153*0Sstevel@tonic-gate cso->cpu_vminfo.pgrrun = CPU_STATS(cp, vm.pgrrun); 3154*0Sstevel@tonic-gate cso->cpu_vminfo.execpgin = CPU_STATS(cp, vm.execpgin); 3155*0Sstevel@tonic-gate cso->cpu_vminfo.execpgout = CPU_STATS(cp, vm.execpgout); 3156*0Sstevel@tonic-gate cso->cpu_vminfo.execfree = CPU_STATS(cp, vm.execfree); 3157*0Sstevel@tonic-gate cso->cpu_vminfo.anonpgin = CPU_STATS(cp, vm.anonpgin); 3158*0Sstevel@tonic-gate cso->cpu_vminfo.anonpgout = CPU_STATS(cp, vm.anonpgout); 3159*0Sstevel@tonic-gate cso->cpu_vminfo.anonfree = CPU_STATS(cp, vm.anonfree); 3160*0Sstevel@tonic-gate cso->cpu_vminfo.fspgin = CPU_STATS(cp, vm.fspgin); 3161*0Sstevel@tonic-gate cso->cpu_vminfo.fspgout = CPU_STATS(cp, vm.fspgout); 3162*0Sstevel@tonic-gate cso->cpu_vminfo.fsfree = CPU_STATS(cp, vm.fsfree); 3163*0Sstevel@tonic-gate 3164*0Sstevel@tonic-gate return (0); 3165*0Sstevel@tonic-gate } 3166