xref: /onnv-gate/usr/src/uts/common/os/cap_util.c (revision 11389:dd00b884e84f)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Support for determining capacity and utilization of performance relevant
 * hardware components in a computer
 *
 * THEORY
 * ------
 * The capacity and utilization of the performance relevant hardware components
 * is needed to be able to optimize performance while minimizing the amount of
 * power used on a system.  The idea is to use hardware performance counters
 * and potentially other means to determine the capacity and utilization of
 * performance relevant hardware components (eg. execution pipeline, cache,
 * memory, etc.) and attribute the utilization to the responsible CPU and the
 * thread running there.
 *
 * This will help characterize the utilization of performance relevant
 * components and how much is used by each CPU and each thread.  With
 * that data, the utilization can be aggregated to all the CPUs sharing each
 * performance relevant hardware component to calculate the total utilization
 * of each component and compare that with the component's capacity to
 * essentially determine the actual hardware load of the component.  The
 * hardware utilization attributed to each running thread can also be
 * aggregated to determine the total hardware utilization of each component to
 * a workload.
 *
 * Once that is done, one can determine how much of each performance relevant
 * hardware component is needed by a given thread or set of threads (eg. a
 * workload) and size up exactly what hardware is needed by the threads and how
 * much.  With this info, we can better place threads among CPUs to match their
 * exact hardware resource needs and potentially lower or raise the power based
 * on their utilization or pack threads onto the fewest hardware components
 * needed and power off any remaining unused components to minimize power
 * without sacrificing performance.
 *
 * IMPLEMENTATION
 * --------------
 * The code has been designed and implemented to make (un)programming and
 * reading the counters for a given CPU as lightweight and fast as possible.
 * This is very important because we need to read and potentially (un)program
 * the counters very often and in performance sensitive code.  Specifically,
 * the counters may need to be (un)programmed during context switch and/or a
 * cyclic handler when there are more counter events to count than existing
 * counters.
 *
 * Consequently, the code has been split up to allow allocating and
 * initializing everything needed to program and read the counters on a given
 * CPU once and make (un)programming and reading the counters for a given CPU
 * not have to allocate/free memory or grab any locks.  To do this, all the
 * state needed to (un)program and read the counters on a CPU is kept per CPU
 * and is made lock free by forcing any code that reads or manipulates the
 * counters or the state needed to (un)program or read the counters to run on
 * the target CPU and disable preemption while running on the target CPU to
 * protect any critical sections. All counter manipulation on the target CPU is
 * happening either from a cross-call to the target CPU or at the same PIL as
 * used by the cross-call subsystem. This guarantees that counter manipulation
 * is not interrupted by cross-calls from other CPUs.
 *
 * The synchronization has been made lock free or as simple as possible for
 * performance and to avoid getting the locking all tangled up when we interpose
 * on the CPC routines that (un)program the counters to manage the counters
 * between the kernel and user on each CPU.  When the user starts using the
 * counters on a given CPU, the kernel will unprogram the counters that it is
 * using on that CPU just before they are programmed for the user.  Then the
 * kernel will program the counters on a given CPU for its own use when the user
 * stops using them.
 *
 * There is a special interaction with DTrace cpc provider (dcpc). Before dcpc
 * enables any probe, it requests to disable and unprogram all counters used for
 * capacity and utilizations. These counters are never re-programmed back until
 * dcpc completes. When all DTrace cpc probes are removed, dcpc notifies CU
 * framework and it re-programs the counters.
 *
 * When a CPU is going offline, its CU counters are unprogrammed and disabled,
 * so that they would not be re-programmed again by some other activity on the
 * CPU that is going offline.
 *
 * The counters are programmed during boot.  However, a flag is available to
 * disable this if necessary (see cu_flag below).  A handler is provided to
 * (un)program the counters during CPU on/offline.  Basic routines are provided
 * to initialize and tear down this module, initialize and tear down any state
 * needed for a given CPU, and (un)program the counters for a given CPU.
 * Lastly, a handler is provided to read the counters and attribute the
 * utilization to the responsible CPU.
 */
#include <sys/types.h>
#include <sys/cmn_err.h>
#include <sys/cpuvar.h>
#include <sys/ddi.h>
#include <sys/disp.h>
#include <sys/sdt.h>
#include <sys/sunddi.h>
#include <sys/thread.h>
#include <sys/pghw.h>
#include <sys/cmt.h>
#include <sys/x_call.h>
#include <sys/cap_util.h>

#include <sys/archsystm.h>
#include <sys/promif.h>

#if defined(__x86)
#include <sys/xc_levels.h>
#endif


/*
 * Default CPU hardware performance counter flags to use for measuring capacity
 * and utilization
 */
#define	CU_CPC_FLAGS_DEFAULT	\
	(CPC_COUNT_USER|CPC_COUNT_SYSTEM|CPC_OVF_NOTIFY_EMT)

/*
 * Possible Flags for controlling this module.
 */
#define	CU_FLAG_ENABLE		1	/* Enable module */
#define	CU_FLAG_READY		2	/* Ready to setup module */
#define	CU_FLAG_ON		4	/* Module is on */

/*
 * pg_cpu kstats calculate utilization rate and maximum utilization rate for
 * some CPUs. The rate is calculated based on data from two subsequent
 * snapshots. When the time between such two snapshots is too small, the
 * resulting rate may have low accuracy, so we only consider snapshots which
 * are separated by SAMPLE_INTERVAL nanoseconds from one another. We do not
 * update the rate if the interval is smaller than that.
 *
 * Use one tenth of a second as the minimum interval for utilization rate
 * calculation.
 *
 * NOTE: The CU_SAMPLE_INTERVAL_MIN should be higher than the scaling factor in
 * the CU_RATE() macro below to guarantee that we never divide by zero.
 *
 * Rate is the number of events per second. The rate is the number of events
 * divided by time and multiplied by the number of nanoseconds in a second. We
 * do not want time to be too small since it will cause large errors in
 * division.
 *
 * We do not want to multiply two large numbers (the instruction count and
 * NANOSEC) either since it may cause integer overflow. So we divide both the
 * numerator and the denominator by the same value.
 *
 * NOTE: The scaling factor below should be less than CU_SAMPLE_INTERVAL_MIN
 * above to guarantee that time divided by this value is always non-zero.
 */
#define	CU_RATE(val, time) \
	(((val) * (NANOSEC / CU_SCALE)) / ((time) / CU_SCALE))

#define	CU_SAMPLE_INTERVAL_MIN	(NANOSEC / 10)

#define	CU_SCALE (CU_SAMPLE_INTERVAL_MIN / 10000)

/*
 * When the time between two kstat reads for the same CPU is less than
 * CU_UPDATE_THRESHOLD use the old counter data and skip updating counter values
 * for the CPU. This helps reduce cross-calls when kstat consumers read data
 * very often or when they read PG utilization data and then CPU utilization
 * data quickly after that.
 */
#define	CU_UPDATE_THRESHOLD (NANOSEC / 10)

/*
 * The IS_HIPIL() macro verifies that the code is executed either from a
 * cross-call or from high-PIL interrupt
 */
#ifdef DEBUG
#define	IS_HIPIL() (getpil() >= XCALL_PIL)
#else
#define	IS_HIPIL()
#endif	/* DEBUG */


typedef void (*cu_cpu_func_t)(uintptr_t, int *);


/*
 * Flags to use for programming CPU hardware performance counters to measure
 * capacity and utilization
 */
int				cu_cpc_flags = CU_CPC_FLAGS_DEFAULT;

/*
 * Initial value used for programming hardware counters
 */
uint64_t			cu_cpc_preset_value = 0;

/*
 * List of CPC event requests for capacity and utilization.
 */
static kcpc_request_list_t	*cu_cpc_reqs = NULL;

/*
 * When a CPU is a member of PG with a sharing relationship that is supported
 * by the capacity/utilization framework, a kstat is created for that CPU and
 * sharing relationship.
 *
 * These kstats are updated one at a time, so we can have a single scratch
 * space to fill the data.
 *
 * CPU counter kstats fields:
 *
 *   cu_cpu_id		CPU ID for this kstat
 *
 *   cu_generation	Generation value that increases whenever any CPU goes
 *			  offline or online. Two kstat snapshots for the same
 *			  CPU may only be compared if they have the same
 *			  generation.
 *
 *   cu_pg_id		PG ID for the relationship described by this kstat
 *
 *   cu_cpu_util	Running value of CPU utilization for the sharing
 *			  relationship
 *
 *   cu_cpu_time_running Total time spent collecting CU data. The time may be
 *			   less than wall time if CU counters were stopped for
 *			   some time.
 *
 *   cu_cpu_time_stopped Total time the CU counters were stopped.
 *
 *   cu_cpu_rate	Utilization rate, expressed in operations per second.
 *
 *   cu_cpu_rate_max	Maximum observed value of utilization rate.
 */
struct cu_cpu_kstat {
	kstat_named_t	cu_cpu_id;
	kstat_named_t	cu_generation;
	kstat_named_t	cu_pg_id;
	kstat_named_t	cu_cpu_util;
	kstat_named_t	cu_cpu_time_running;
	kstat_named_t	cu_cpu_time_stopped;
	kstat_named_t	cu_cpu_rate;
	kstat_named_t	cu_cpu_rate_max;
} cu_cpu_kstat = {
	{ "id",				KSTAT_DATA_UINT32 },
	{ "generation",			KSTAT_DATA_UINT32 },
	{ "pg_id",			KSTAT_DATA_LONG },
	{ "hw_util",			KSTAT_DATA_UINT64 },
	{ "hw_util_time_running",	KSTAT_DATA_UINT64 },
	{ "hw_util_time_stopped",	KSTAT_DATA_UINT64 },
	{ "hw_util_rate",		KSTAT_DATA_UINT64 },
	{ "hw_util_rate_max",		KSTAT_DATA_UINT64 },
};

/*
 * Flags for controlling this module
 */
uint_t				cu_flags = CU_FLAG_ENABLE;

/*
 * Error return value for cu_init() since it can't return anything to be called
 * from mp_init_tbl[] (:-(
 */
static int			cu_init_error = 0;

hrtime_t			cu_sample_interval_min = CU_SAMPLE_INTERVAL_MIN;

hrtime_t			cu_update_threshold = CU_UPDATE_THRESHOLD;

static kmutex_t			pg_cpu_kstat_lock;


/*
 * Forward declaration of interface routines
 */
void		cu_disable(void);
void		cu_enable(void);
void		cu_init(void);
void		cu_cpc_program(cpu_t *cp, int *err);
void		cu_cpc_unprogram(cpu_t *cp, int *err);
int		cu_cpu_update(struct cpu *cp, boolean_t move_to);
void		cu_pg_update(pghw_t *pg);


/*
 * Forward declaration of private routines
 */
static int	cu_cpc_init(cpu_t *cp, kcpc_request_list_t *reqs, int nreqs);
static void	cu_cpc_program_xcall(uintptr_t arg, int *err);
static int	cu_cpc_req_add(char *event, kcpc_request_list_t *reqs,
    int nreqs, cu_cntr_stats_t *stats, int kmem_flags, int *nevents);
static int	cu_cpu_callback(cpu_setup_t what, int id, void *arg);
static void	cu_cpu_disable(cpu_t *cp);
static void	cu_cpu_enable(cpu_t *cp);
static int	cu_cpu_init(cpu_t *cp, kcpc_request_list_t *reqs);
static int	cu_cpu_fini(cpu_t *cp);
static void	cu_cpu_kstat_create(pghw_t *pg, cu_cntr_info_t *cntr_info);
static int	cu_cpu_kstat_update(kstat_t *ksp, int rw);
static int	cu_cpu_run(cpu_t *cp, cu_cpu_func_t func, uintptr_t arg);
static int	cu_cpu_update_stats(cu_cntr_stats_t *stats,
    uint64_t cntr_value);
static void cu_cpu_info_detach_xcall(void);

/*
 * Disable or enable Capacity Utilization counters on all CPUs.
 */
void
cu_disable(void)
{
	cpu_t *cp;

	ASSERT(MUTEX_HELD(&cpu_lock));

	cp = cpu_active;
	do {
		if (!(cp->cpu_flags & CPU_OFFLINE))
			cu_cpu_disable(cp);
	} while ((cp = cp->cpu_next_onln) != cpu_active);
}


void
cu_enable(void)
{
	cpu_t *cp;

	ASSERT(MUTEX_HELD(&cpu_lock));

	cp = cpu_active;
	do {
		if (!(cp->cpu_flags & CPU_OFFLINE))
			cu_cpu_enable(cp);
	} while ((cp = cp->cpu_next_onln) != cpu_active);
}


/*
 * Setup capacity and utilization support
 */
void
cu_init(void)
{
	cpu_t	*cp;

	cu_init_error = 0;
	if (!(cu_flags & CU_FLAG_ENABLE) || (cu_flags & CU_FLAG_ON)) {
		cu_init_error = -1;
		return;
	}

	if (kcpc_init() != 0) {
		cu_init_error = -2;
		return;
	}

	/*
	 * Can't measure hardware capacity and utilization without CPU
	 * hardware performance counters
	 */
	if (cpc_ncounters <= 0) {
		cu_init_error = -3;
		return;
	}

	/*
	 * Setup CPC event request queue
	 */
	cu_cpc_reqs = kcpc_reqs_init(cpc_ncounters, KM_SLEEP);

	mutex_enter(&cpu_lock);

	/*
	 * Mark flags to say that module is ready to be setup
	 */
	cu_flags |= CU_FLAG_READY;

	cp = cpu_active;
	do {
		/*
		 * Allocate and setup state needed to measure capacity and
		 * utilization
		 */
		if (cu_cpu_init(cp, cu_cpc_reqs) != 0)
			cu_init_error = -5;

		/*
		 * Reset list of counter event requests so its space can be
		 * reused for a different set of requests for next CPU
		 */
		(void) kcpc_reqs_reset(cu_cpc_reqs);

		cp = cp->cpu_next_onln;
	} while (cp != cpu_active);

	/*
	 * Mark flags to say that module is on now and counters are ready to be
	 * programmed on all active CPUs
	 */
	cu_flags |= CU_FLAG_ON;

	/*
	 * Program counters on currently active CPUs
	 */
	cp = cpu_active;
	do {
		if (cu_cpu_run(cp, cu_cpc_program_xcall,
		    (uintptr_t)B_FALSE) != 0)
			cu_init_error = -6;

		cp = cp->cpu_next_onln;
	} while (cp != cpu_active);

	/*
	 * Register callback for CPU state changes to enable and disable
	 * CPC counters as CPUs come on and offline
	 */
	register_cpu_setup_func(cu_cpu_callback, NULL);

	mutex_exit(&cpu_lock);
}


/*
 * Return number of counter events needed to measure capacity and utilization
 * for specified CPU and fill in list of CPC requests with each counter event
 * needed if list where to add CPC requests is given
 *
 * NOTE: Use KM_NOSLEEP for kmem_{,z}alloc() since cpu_lock is held and free
 *	 everything that has been successfully allocated if any memory
 *	 allocation fails
 */
static int
cu_cpc_init(cpu_t *cp, kcpc_request_list_t *reqs, int nreqs)
{
	group_t		*cmt_pgs;
	cu_cntr_info_t	**cntr_info_array;
	cpu_pg_t	*cpu_pgs;
	cu_cpu_info_t	*cu_cpu_info;
	pg_cmt_t	*pg_cmt;
	pghw_t		*pg_hw;
	cu_cntr_stats_t	*stats;
	int		nevents;
	pghw_type_t	pg_hw_type;
	group_iter_t	iter;

	ASSERT(MUTEX_HELD(&cpu_lock));

	/*
	 * There has to be a target CPU for this
	 */
	if (cp == NULL)
		return (-1);

	/*
	 * Return 0 when CPU doesn't belong to any group
	 */
	cpu_pgs = cp->cpu_pg;
	if (cpu_pgs == NULL || GROUP_SIZE(&cpu_pgs->cmt_pgs) < 1)
		return (0);

	cmt_pgs = &cpu_pgs->cmt_pgs;
	cu_cpu_info = cp->cpu_cu_info;

	/*
	 * Grab counter statistics and info
	 */
	if (reqs == NULL) {
		stats = NULL;
		cntr_info_array = NULL;
	} else {
		if (cu_cpu_info == NULL || cu_cpu_info->cu_cntr_stats == NULL)
			return (-2);

		stats = cu_cpu_info->cu_cntr_stats;
		cntr_info_array = cu_cpu_info->cu_cntr_info;
	}

	/*
	 * See whether platform (or processor) specific code knows which CPC
	 * events to request, etc. are needed to measure hardware capacity and
	 * utilization on this machine
	 */
	nevents = cu_plat_cpc_init(cp, reqs, nreqs);
	if (nevents >= 0)
		return (nevents);

	/*
	 * Let common code decide which CPC events to request, etc. to measure
	 * capacity and utilization since platform (or processor) specific does
	 * not know....
	 *
	 * Walk CPU's PG lineage and do following:
	 *
	 * - Setup CPC request, counter info, and stats needed for each counter
	 *   event to measure capacity and and utilization for each of CPU's PG
	 *   hardware sharing relationships
	 *
	 * - Create PG CPU kstats to export capacity and utilization for each PG
	 */
	nevents = 0;
	group_iter_init(&iter);
	while ((pg_cmt = group_iterate(cmt_pgs, &iter)) != NULL) {
		cu_cntr_info_t	*cntr_info;
		int		nevents_save;
		int		nstats;

		pg_hw = (pghw_t *)pg_cmt;
		pg_hw_type = pg_hw->pghw_hw;
		nevents_save = nevents;
		nstats = 0;

		switch (pg_hw_type) {
		case PGHW_IPIPE:
			if (cu_cpc_req_add("PAPI_tot_ins", reqs, nreqs, stats,
			    KM_NOSLEEP, &nevents) != 0)
				continue;
			nstats = 1;
			break;

		case PGHW_FPU:
			if (cu_cpc_req_add("PAPI_fp_ins", reqs, nreqs, stats,
			    KM_NOSLEEP, &nevents) != 0)
				continue;
			nstats = 1;
			break;

		default:
			/*
			 * Don't measure capacity and utilization for this kind
			 * of PG hardware relationship so skip to next PG in
			 * CPU's PG lineage
			 */
			continue;
		}

		cntr_info = cntr_info_array[pg_hw_type];

		/*
		 * Nothing to measure for this hardware sharing relationship
		 */
		if (nevents - nevents_save == 0) {
			if (cntr_info != NULL)
				kmem_free(cntr_info, sizeof (cu_cntr_info_t));
				cntr_info_array[pg_hw_type] = NULL;
			continue;
		}

		/*
		 * Fill in counter info for this PG hardware relationship
		 */
		if (cntr_info == NULL) {
			cntr_info = kmem_zalloc(sizeof (cu_cntr_info_t),
			    KM_NOSLEEP);
			if (cntr_info == NULL)
				continue;
			cntr_info_array[pg_hw_type] = cntr_info;
		}
		cntr_info->ci_cpu = cp;
		cntr_info->ci_pg = pg_hw;
		cntr_info->ci_stats = &stats[nevents_save];
		cntr_info->ci_nstats = nstats;

		/*
		 * Create PG CPU kstats for this hardware relationship
		 */
		cu_cpu_kstat_create(pg_hw, cntr_info);
	}

	return (nevents);
}


/*
 * Program counters for capacity and utilization on given CPU
 *
 * If any of the following conditions is true, the counters are not programmed:
 *
 * - CU framework is disabled
 * - The cpu_cu_info field of the cpu structure is NULL
 * - DTrace is active
 * - Counters are programmed already
 * - Counters are disabled (by calls to cu_cpu_disable())
 */
void
cu_cpc_program(cpu_t *cp, int *err)
{
	cu_cpc_ctx_t	*cpu_ctx;
	kcpc_ctx_t	*ctx;
	cu_cpu_info_t	*cu_cpu_info;

	ASSERT(IS_HIPIL());
	/*
	 * Should be running on given CPU. We disable preemption to keep CPU
	 * from disappearing and make sure flags and CPC context don't change
	 * from underneath us
	 */
	kpreempt_disable();
	ASSERT(cp == CPU);

	/*
	 * Module not ready to program counters
	 */
	if (!(cu_flags & CU_FLAG_ON)) {
		*err = -1;
		kpreempt_enable();
		return;
	}

	if (cp == NULL) {
		*err = -2;
		kpreempt_enable();
		return;
	}

	cu_cpu_info = cp->cpu_cu_info;
	if (cu_cpu_info == NULL) {
		*err = -3;
		kpreempt_enable();
		return;
	}

	/*
	 * If DTrace CPC is active or counters turned on already or are
	 * disabled, just return.
	 */
	if (dtrace_cpc_in_use || (cu_cpu_info->cu_flag & CU_CPU_CNTRS_ON) ||
	    cu_cpu_info->cu_disabled) {
		*err = 1;
		kpreempt_enable();
		return;
	}

	if ((CPU->cpu_cpc_ctx != NULL) &&
	    !(CPU->cpu_cpc_ctx->kc_flags & KCPC_CTX_INVALID_STOPPED)) {
		*err = -4;
		kpreempt_enable();
		return;
	}

	/*
	 * Get CPU's CPC context needed for capacity and utilization
	 */
	cpu_ctx = &cu_cpu_info->cu_cpc_ctx;
	ASSERT(cpu_ctx != NULL);
	ASSERT(cpu_ctx->nctx >= 0);

	ASSERT(cpu_ctx->ctx_ptr_array == NULL || cpu_ctx->ctx_ptr_array_sz > 0);
	ASSERT(cpu_ctx->nctx <= cpu_ctx->ctx_ptr_array_sz);
	if (cpu_ctx->nctx <= 0 || cpu_ctx->ctx_ptr_array == NULL ||
	    cpu_ctx->ctx_ptr_array_sz <= 0) {
		*err = -5;
		kpreempt_enable();
		return;
	}

	/*
	 * Increment index in CPU's CPC context info to point at next context
	 * to program
	 *
	 * NOTE: Do this now instead of after programming counters to ensure
	 *	 that index will always point at *current* context so we will
	 *	 always be able to unprogram *current* context if necessary
	 */
	cpu_ctx->cur_index = (cpu_ctx->cur_index + 1) % cpu_ctx->nctx;

	ctx = cpu_ctx->ctx_ptr_array[cpu_ctx->cur_index];

	/*
	 * Clear KCPC_CTX_INVALID and KCPC_CTX_INVALID_STOPPED from CPU's CPC
	 * context before programming counters
	 *
	 * Context is marked with KCPC_CTX_INVALID_STOPPED when context is
	 * unprogrammed and may be marked with KCPC_CTX_INVALID when
	 * kcpc_invalidate_all() is called by cpustat(1M) and dtrace CPC to
	 * invalidate all CPC contexts before they take over all the counters.
	 *
	 * This isn't necessary since these flags are only used for thread bound
	 * CPC contexts not CPU bound CPC contexts like ones used for capacity
	 * and utilization.
	 *
	 * There is no need to protect the flag update since no one is using
	 * this context now.
	 */
	ctx->kc_flags &= ~(KCPC_CTX_INVALID | KCPC_CTX_INVALID_STOPPED);

	/*
	 * Program counters on this CPU
	 */
	kcpc_program(ctx, B_FALSE, B_FALSE);

	cp->cpu_cpc_ctx = ctx;

	/*
	 * Set state in CPU structure to say that CPU's counters are programmed
	 * for capacity and utilization now and that they are transitioning from
	 * off to on state. This will cause cu_cpu_update to update stop times
	 * for all programmed counters.
	 */
	cu_cpu_info->cu_flag |= CU_CPU_CNTRS_ON | CU_CPU_CNTRS_OFF_ON;

	/*
	 * Update counter statistics
	 */
	(void) cu_cpu_update(cp, B_FALSE);

	cu_cpu_info->cu_flag &= ~CU_CPU_CNTRS_OFF_ON;

	*err = 0;
	kpreempt_enable();
}


/*
 * Cross call wrapper routine for cu_cpc_program()
 *
 * Checks to make sure that counters on CPU aren't being used by someone else
 * before calling cu_cpc_program() since cu_cpc_program() needs to assert that
 * nobody else is using the counters to catch and prevent any broken code.
 * Also, this check needs to happen on the target CPU since the CPU's CPC
 * context can only be changed while running on the CPU.
 *
 * If the first argument is TRUE, cu_cpc_program_xcall also checks that there is
 * no valid thread bound cpc context. This is important to check to prevent
 * re-programming thread counters with CU counters when CPU is coming on-line.
 */
static void
cu_cpc_program_xcall(uintptr_t arg, int *err)
{
	boolean_t	avoid_thread_context = (boolean_t)arg;

	kpreempt_disable();

	if (CPU->cpu_cpc_ctx != NULL &&
	    !(CPU->cpu_cpc_ctx->kc_flags & KCPC_CTX_INVALID_STOPPED)) {
		*err = -100;
		kpreempt_enable();
		return;
	}

	if (avoid_thread_context && (curthread->t_cpc_ctx != NULL) &&
	    !(curthread->t_cpc_ctx->kc_flags & KCPC_CTX_INVALID_STOPPED)) {
		*err = -200;
		kpreempt_enable();
		return;
	}

	cu_cpc_program(CPU, err);
	kpreempt_enable();
}


/*
 * Unprogram counters for capacity and utilization on given CPU
 * This function should be always executed on the target CPU at high PIL
 */
void
cu_cpc_unprogram(cpu_t *cp, int *err)
{
	cu_cpc_ctx_t	*cpu_ctx;
	kcpc_ctx_t	*ctx;
	cu_cpu_info_t	*cu_cpu_info;

	ASSERT(IS_HIPIL());
	/*
	 * Should be running on given CPU with preemption disabled to keep CPU
	 * from disappearing and make sure flags and CPC context don't change
	 * from underneath us
	 */
	kpreempt_disable();
	ASSERT(cp == CPU);

	/*
	 * Module not on
	 */
	if (!(cu_flags & CU_FLAG_ON)) {
		*err = -1;
		kpreempt_enable();
		return;
	}

	cu_cpu_info = cp->cpu_cu_info;
	if (cu_cpu_info == NULL) {
		*err = -3;
		kpreempt_enable();
		return;
	}

	/*
	 * Counters turned off already
	 */
	if (!(cu_cpu_info->cu_flag & CU_CPU_CNTRS_ON)) {
		*err = 1;
		kpreempt_enable();
		return;
	}

	/*
	 * Update counter statistics
	 */
	(void) cu_cpu_update(cp, B_FALSE);

	/*
	 * Get CPU's CPC context needed for capacity and utilization
	 */
	cpu_ctx = &cu_cpu_info->cu_cpc_ctx;
	if (cpu_ctx->nctx <= 0 || cpu_ctx->ctx_ptr_array == NULL ||
	    cpu_ctx->ctx_ptr_array_sz <= 0) {
		*err = -5;
		kpreempt_enable();
		return;
	}
	ctx = cpu_ctx->ctx_ptr_array[cpu_ctx->cur_index];

	/*
	 * CPU's CPC context should be current capacity and utilization CPC
	 * context
	 */
	ASSERT(cp->cpu_cpc_ctx == ctx);
	if (cp->cpu_cpc_ctx != ctx) {
		*err = -6;
		kpreempt_enable();
		return;
	}

	/*
	 * Unprogram counters on CPU.
	 */
	kcpc_unprogram(ctx, B_FALSE);

	ASSERT(ctx->kc_flags & KCPC_CTX_INVALID_STOPPED);

	/*
	 * Unset state in CPU structure saying that CPU's counters are
	 * programmed
	 */
	cp->cpu_cpc_ctx = NULL;
	cu_cpu_info->cu_flag &= ~CU_CPU_CNTRS_ON;

	*err = 0;
	kpreempt_enable();
}


/*
 * Add given counter event to list of CPC requests
 */
static int
cu_cpc_req_add(char *event, kcpc_request_list_t *reqs, int nreqs,
    cu_cntr_stats_t *stats, int kmem_flags, int *nevents)
{
	int	n;
	int	retval;
	uint_t  flags;

	/*
	 * Return error when no counter event specified, counter event not
	 * supported by CPC's PCBE, or number of events not given
	 */
	if (event == NULL || kcpc_event_supported(event) == B_FALSE ||
	    nevents == NULL)
		return (-1);

	n = *nevents;

	/*
	 * Only count number of counter events needed if list
	 * where to add CPC requests not given
	 */
	if (reqs == NULL) {
		n++;
		*nevents = n;
		return (-3);
	}

	/*
	 * Return error when stats not given or not enough room on list of CPC
	 * requests for more counter events
	 */
	if (stats == NULL || (nreqs <= 0 && n >= nreqs))
		return (-4);

	/*
	 * Use flags in cu_cpc_flags to program counters and enable overflow
	 * interrupts/traps (unless PCBE can't handle overflow interrupts) so
	 * PCBE can catch counters before they wrap to hopefully give us an
	 * accurate (64-bit) virtualized counter
	 */
	flags = cu_cpc_flags;
	if ((kcpc_pcbe_capabilities() & CPC_CAP_OVERFLOW_INTERRUPT) == 0)
		flags &= ~CPC_OVF_NOTIFY_EMT;

	/*
	 * Add CPC request to list
	 */
	retval = kcpc_reqs_add(reqs, event, cu_cpc_preset_value,
	    flags, 0, NULL, &stats[n], kmem_flags);

	if (retval != 0)
		return (-5);

	n++;
	*nevents = n;
	return (0);
}

static void
cu_cpu_info_detach_xcall(void)
{
	ASSERT(IS_HIPIL());

	CPU->cpu_cu_info = NULL;
}


/*
 * Enable or disable collection of capacity/utilization data for a current CPU.
 * Counters are enabled if 'on' argument is True and disabled if it is False.
 * This function should be always executed at high PIL
 */
static void
cu_cpc_trigger(uintptr_t arg1, uintptr_t arg2)
{
	cpu_t		*cp = (cpu_t *)arg1;
	boolean_t	on = (boolean_t)arg2;
	int		error;
	cu_cpu_info_t	*cu_cpu_info;

	ASSERT(IS_HIPIL());
	kpreempt_disable();
	ASSERT(cp == CPU);

	if (!(cu_flags & CU_FLAG_ON)) {
		kpreempt_enable();
		return;
	}

	cu_cpu_info = cp->cpu_cu_info;
	if (cu_cpu_info == NULL) {
		kpreempt_enable();
		return;
	}

	ASSERT(!cu_cpu_info->cu_disabled ||
	    !(cu_cpu_info->cu_flag & CU_CPU_CNTRS_ON));

	if (on) {
		/*
		 * Decrement the cu_disabled counter.
		 * Once it drops to zero, call cu_cpc_program.
		 */
		if (cu_cpu_info->cu_disabled > 0)
			cu_cpu_info->cu_disabled--;
		if (cu_cpu_info->cu_disabled == 0)
			cu_cpc_program(CPU, &error);
	} else if (cu_cpu_info->cu_disabled++ == 0) {
		/*
		 * This is the first attempt to disable CU, so turn it off
		 */
		cu_cpc_unprogram(cp, &error);
		ASSERT(!(cu_cpu_info->cu_flag & CU_CPU_CNTRS_ON));
	}

	kpreempt_enable();
}


/*
 * Callback for changes in CPU states
 * Used to enable or disable hardware performance counters on CPUs that are
 * turned on or off
 *
 * NOTE: cpc should be programmed/unprogrammed while running on the target CPU.
 * We have to use thread_affinity_set to hop to the right CPU because these
 * routines expect cpu_lock held, so we can't cross-call other CPUs while
 * holding CPU lock.
 */
static int
/* LINTED E_FUNC_ARG_UNUSED */
cu_cpu_callback(cpu_setup_t what, int id, void *arg)
{
	cpu_t	*cp;
	int	retval = 0;

	ASSERT(MUTEX_HELD(&cpu_lock));

	if (!(cu_flags & CU_FLAG_ON))
		return (-1);

	cp = cpu_get(id);
	if (cp == NULL)
		return (-2);

	switch (what) {
	case CPU_ON:
		/*
		 * Setup counters on CPU being turned on
		 */
		retval = cu_cpu_init(cp, cu_cpc_reqs);

		/*
		 * Reset list of counter event requests so its space can be
		 * reused for a different set of requests for next CPU
		 */
		(void) kcpc_reqs_reset(cu_cpc_reqs);
		break;
	case CPU_INTR_ON:
		/*
		 * Setup counters on CPU being turned on.
		 */
		retval = cu_cpu_run(cp, cu_cpc_program_xcall,
		    (uintptr_t)B_TRUE);
		break;
	case CPU_OFF:
		/*
		 * Disable counters on CPU being turned off. Counters will not
		 * be re-enabled on this CPU until it comes back online.
		 */
		cu_cpu_disable(cp);
		ASSERT(!CU_CPC_ON(cp));
		retval = cu_cpu_fini(cp);
		break;
	default:
		break;
	}
	return (retval);
}


/*
 * Disable or enable Capacity Utilization counters on a given CPU. This function
 * can be called from any CPU to disable counters on the given CPU.
 */
static void
cu_cpu_disable(cpu_t *cp)
{
	cpu_call(cp, cu_cpc_trigger, (uintptr_t)cp, (uintptr_t)B_FALSE);
}


static void
cu_cpu_enable(cpu_t *cp)
{
	cpu_call(cp, cu_cpc_trigger, (uintptr_t)cp, (uintptr_t)B_TRUE);
}


/*
 * Setup capacity and utilization support for given CPU
 *
 * NOTE: Use KM_NOSLEEP for kmem_{,z}alloc() since cpu_lock is held and free
 *	 everything that has been successfully allocated including cpu_cu_info
 *	if any memory allocation fails
 */
static int
cu_cpu_init(cpu_t *cp, kcpc_request_list_t *reqs)
{
	kcpc_ctx_t	**ctx_ptr_array;
	size_t		ctx_ptr_array_sz;
	cu_cpc_ctx_t	*cpu_ctx;
	cu_cpu_info_t	*cu_cpu_info;
	int		n;

	/*
	 * cpu_lock should be held and protect against CPU going away and races
	 * with cu_{init,fini,cpu_fini}()
	 */
	ASSERT(MUTEX_HELD(&cpu_lock));

	/*
	 * Return if not ready to setup counters yet
	 */
	if (!(cu_flags & CU_FLAG_READY))
		return (-1);

	if (cp->cpu_cu_info == NULL) {
		cp->cpu_cu_info = kmem_zalloc(sizeof (cu_cpu_info_t),
		    KM_NOSLEEP);
		if (cp->cpu_cu_info == NULL)
			return (-2);
	}

	/*
	 * Get capacity and utilization CPC context for CPU and check to see
	 * whether it has been setup already
	 */
	cu_cpu_info = cp->cpu_cu_info;
	cu_cpu_info->cu_cpu = cp;
	cu_cpu_info->cu_disabled = dtrace_cpc_in_use ? 1 : 0;

	cpu_ctx = &cu_cpu_info->cu_cpc_ctx;
	if (cpu_ctx->nctx > 0 && cpu_ctx->ctx_ptr_array != NULL &&
	    cpu_ctx->ctx_ptr_array_sz > 0) {
		return (1);
	}

	/*
	 * Should have no contexts since it hasn't been setup already
	 */
	ASSERT(cpu_ctx->nctx == 0 && cpu_ctx->ctx_ptr_array == NULL &&
	    cpu_ctx->ctx_ptr_array_sz == 0);

	/*
	 * Determine how many CPC events needed to measure capacity and
	 * utilization for this CPU, allocate space for counter statistics for
	 * each event, and fill in list of CPC event requests with corresponding
	 * counter stats for each request to make attributing counter data
	 * easier later....
	 */
	n = cu_cpc_init(cp, NULL, 0);
	if (n <= 0) {
		(void) cu_cpu_fini(cp);
		return (-3);
	}

	cu_cpu_info->cu_cntr_stats = kmem_zalloc(n * sizeof (cu_cntr_stats_t),
	    KM_NOSLEEP);
	if (cu_cpu_info->cu_cntr_stats == NULL) {
		(void) cu_cpu_fini(cp);
		return (-4);
	}

	cu_cpu_info->cu_ncntr_stats = n;

	n = cu_cpc_init(cp, reqs, n);
	if (n <= 0) {
		(void) cu_cpu_fini(cp);
		return (-5);
	}

	/*
	 * Create CPC context with given requests
	 */
	ctx_ptr_array = NULL;
	ctx_ptr_array_sz = 0;
	n = kcpc_cpu_ctx_create(cp, reqs, KM_NOSLEEP, &ctx_ptr_array,
	    &ctx_ptr_array_sz);
	if (n <= 0) {
		(void) cu_cpu_fini(cp);
		return (-6);
	}

	/*
	 * Should have contexts
	 */
	ASSERT(n > 0 && ctx_ptr_array != NULL && ctx_ptr_array_sz > 0);
	if (ctx_ptr_array == NULL || ctx_ptr_array_sz <= 0) {
		(void) cu_cpu_fini(cp);
		return (-7);
	}

	/*
	 * Fill in CPC context info for CPU needed for capacity and utilization
	 */
	cpu_ctx->cur_index = 0;
	cpu_ctx->nctx = n;
	cpu_ctx->ctx_ptr_array = ctx_ptr_array;
	cpu_ctx->ctx_ptr_array_sz = ctx_ptr_array_sz;
	return (0);
}

/*
 * Tear down capacity and utilization support for given CPU
 */
static int
cu_cpu_fini(cpu_t *cp)
{
	kcpc_ctx_t	*ctx;
	cu_cpc_ctx_t	*cpu_ctx;
	cu_cpu_info_t	*cu_cpu_info;
	int		i;
	pghw_type_t	pg_hw_type;

	/*
	 * cpu_lock should be held and protect against CPU going away and races
	 * with cu_{init,fini,cpu_init}()
	 */
	ASSERT(MUTEX_HELD(&cpu_lock));

	/*
	 * Have to at least be ready to setup counters to have allocated
	 * anything that needs to be deallocated now
	 */
	if (!(cu_flags & CU_FLAG_READY))
		return (-1);

	/*
	 * Nothing to do if CPU's capacity and utilization info doesn't exist
	 */
	cu_cpu_info = cp->cpu_cu_info;
	if (cu_cpu_info == NULL)
		return (1);

	/*
	 * Tear down any existing kstats and counter info for each hardware
	 * sharing relationship
	 */
	for (pg_hw_type = PGHW_START; pg_hw_type < PGHW_NUM_COMPONENTS;
	    pg_hw_type++) {
		cu_cntr_info_t	*cntr_info;

		cntr_info = cu_cpu_info->cu_cntr_info[pg_hw_type];
		if (cntr_info == NULL)
			continue;

		if (cntr_info->ci_kstat != NULL) {
			kstat_delete(cntr_info->ci_kstat);
			cntr_info->ci_kstat = NULL;
		}
		kmem_free(cntr_info, sizeof (cu_cntr_info_t));
	}

	/*
	 * Free counter statistics for CPU
	 */
	ASSERT(cu_cpu_info->cu_cntr_stats == NULL ||
	    cu_cpu_info->cu_ncntr_stats > 0);
	if (cu_cpu_info->cu_cntr_stats != NULL &&
	    cu_cpu_info->cu_ncntr_stats > 0) {
		kmem_free(cu_cpu_info->cu_cntr_stats,
		    cu_cpu_info->cu_ncntr_stats * sizeof (cu_cntr_stats_t));
		cu_cpu_info->cu_cntr_stats = NULL;
		cu_cpu_info->cu_ncntr_stats = 0;
	}

	/*
	 * Get capacity and utilization CPC contexts for given CPU and check to
	 * see whether they have been freed already
	 */
	cpu_ctx = &cu_cpu_info->cu_cpc_ctx;
	if (cpu_ctx != NULL && cpu_ctx->ctx_ptr_array != NULL &&
	    cpu_ctx->ctx_ptr_array_sz > 0) {
		/*
		 * Free CPC contexts for given CPU
		 */
		for (i = 0; i < cpu_ctx->nctx; i++) {
			ctx = cpu_ctx->ctx_ptr_array[i];
			if (ctx == NULL)
				continue;
			kcpc_free(ctx, 0);
		}

		/*
		 * Free CPC context pointer array
		 */
		kmem_free(cpu_ctx->ctx_ptr_array, cpu_ctx->ctx_ptr_array_sz);

		/*
		 * Zero CPC info for CPU
		 */
		bzero(cpu_ctx, sizeof (cu_cpc_ctx_t));
	}

	/*
	 * Set cp->cpu_cu_info pointer to NULL. Go through cross-call to ensure
	 * that no one is going to access the cpu_cu_info whicch we are going to
	 * free.
	 */
	if (cpu_is_online(cp))
		cpu_call(cp, (cpu_call_func_t)cu_cpu_info_detach_xcall, 0, 0);
	else
		cp->cpu_cu_info = NULL;

	/*
	 * Free CPU's capacity and utilization info
	 */
	kmem_free(cu_cpu_info, sizeof (cu_cpu_info_t));

	return (0);
}

/*
 * Create capacity & utilization kstats for given PG CPU hardware sharing
 * relationship
 */
static void
cu_cpu_kstat_create(pghw_t *pg, cu_cntr_info_t *cntr_info)
{
	char		*class, *sh_name;
	kstat_t		*ks;

	/*
	 * Just return when no counter info or CPU
	 */
	if (cntr_info == NULL || cntr_info->ci_cpu == NULL)
		return;

	/*
	 * Get the class name from the leaf PG that this CPU belongs to.
	 * If there are no PGs, just use the default class "cpu".
	 */
	class = pg ? pghw_type_string(pg->pghw_hw) : "cpu";
	sh_name = pg ? pghw_type_shortstring(pg->pghw_hw) : "cpu";

	if ((ks = kstat_create_zone("pg_cpu", cntr_info->ci_cpu->cpu_id,
	    sh_name, class, KSTAT_TYPE_NAMED,
	    sizeof (cu_cpu_kstat) / sizeof (kstat_named_t),
	    KSTAT_FLAG_VIRTUAL, GLOBAL_ZONEID)) == NULL)
		return;

	ks->ks_lock = &pg_cpu_kstat_lock;
	ks->ks_data = &cu_cpu_kstat;
	ks->ks_update = cu_cpu_kstat_update;

	ks->ks_private = cntr_info;
	cntr_info->ci_kstat = ks;
	kstat_install(cntr_info->ci_kstat);
}


/*
 * Propagate values from CPU capacity & utilization stats to kstats
 */
static int
cu_cpu_kstat_update(kstat_t *ksp, int rw)
{
	cpu_t		*cp;
	cu_cntr_info_t	*cntr_info = ksp->ks_private;
	struct cu_cpu_kstat	*kstat = &cu_cpu_kstat;
	pghw_t		*pg;
	cu_cntr_stats_t	*stats;

	if (rw == KSTAT_WRITE)
		return (EACCES);

	kpreempt_disable();

	/*
	 * Update capacity and utilization statistics needed for CPU's PG (CPU)
	 * kstats
	 */
	cp = cntr_info->ci_cpu;
	(void) cu_cpu_update(cp, B_TRUE);

	pg = cntr_info->ci_pg;
	stats = cntr_info->ci_stats;
	kstat->cu_cpu_id.value.ui32 = cp->cpu_id;
	kstat->cu_generation.value.ui32 = cp->cpu_generation;
	if (pg == NULL)
		kstat->cu_pg_id.value.l = -1;
	else
		kstat->cu_pg_id.value.l = pg->pghw_pg.pg_id;

	kstat->cu_cpu_util.value.ui64 = stats->cs_value_total;
	kstat->cu_cpu_rate.value.ui64 = stats->cs_rate;
	kstat->cu_cpu_rate_max.value.ui64 = stats->cs_rate_max;
	kstat->cu_cpu_time_running.value.ui64 = stats->cs_time_running;
	kstat->cu_cpu_time_stopped.value.ui64 = stats->cs_time_stopped;
	/*
	 * Counters are stopped now, so the cs_time_stopped was last
	 * updated at cs_time_start time. Add the time passed since then
	 * to the stopped time.
	 */
	if (!(cp->cpu_cu_info->cu_flag & CU_CPU_CNTRS_ON))
		kstat->cu_cpu_time_stopped.value.ui64 +=
		    gethrtime() - stats->cs_time_start;

	kpreempt_enable();

	return (0);
}

/*
 * Run specified function with specified argument on a given CPU and return
 * whatever the function returns
 */
static int
cu_cpu_run(cpu_t *cp, cu_cpu_func_t func, uintptr_t arg)
{
	int error = 0;

	/*
	 * cpu_call() will call func on the CPU specified with given argument
	 * and return func's return value in last argument
	 */
	cpu_call(cp, (cpu_call_func_t)func, arg, (uintptr_t)&error);
	return (error);
}


/*
 * Update counter statistics on a given CPU.
 *
 * If move_to argument is True, execute the function on the CPU specified
 * Otherwise, assume that it is already runninng on the right CPU
 *
 * If move_to is specified, the caller should hold cpu_lock or have preemption
 * disabled. Otherwise it is up to the caller to guarantee that things do not
 * change in the process.
 */
int
cu_cpu_update(struct cpu *cp, boolean_t move_to)
{
	int	retval;
	cu_cpu_info_t	*cu_cpu_info = cp->cpu_cu_info;
	hrtime_t	time_snap;

	ASSERT(!move_to || MUTEX_HELD(&cpu_lock) || curthread->t_preempt > 0);

	/*
	 * Nothing to do if counters are not programmed
	 */
	if (!(cu_flags & CU_FLAG_ON) ||
	    (cu_cpu_info == NULL) ||
	    !(cu_cpu_info->cu_flag & CU_CPU_CNTRS_ON))
		return (0);

	/*
	 * Don't update CPU statistics if it was updated recently
	 * and provide old results instead
	 */
	time_snap = gethrtime();
	if ((time_snap - cu_cpu_info->cu_sample_time) < cu_update_threshold) {
		DTRACE_PROBE1(cu__drop__sample, cpu_t *, cp);
		return (0);
	}

	cu_cpu_info->cu_sample_time = time_snap;

	/*
	 * CPC counter should be read on the CPU that is running the counter. We
	 * either have to move ourselves to the target CPU or insure that we
	 * already run there.
	 *
	 * We use cross-call to the target CPU to execute kcpc_read() and
	 * cu_cpu_update_stats() there.
	 */
	retval = 0;
	if (move_to)
		(void) cu_cpu_run(cp, (cu_cpu_func_t)kcpc_read,
		    (uintptr_t)cu_cpu_update_stats);
	else {
		retval = kcpc_read((kcpc_update_func_t)cu_cpu_update_stats);
		/*
		 * Offset negative return value by -10 so we can distinguish it
		 * from error return values of this routine vs kcpc_read()
		 */
		if (retval < 0)
			retval -= 10;
	}

	return (retval);
}


/*
 * Update CPU counter statistics for current CPU.
 * This function may be called from a cross-call
 */
static int
cu_cpu_update_stats(cu_cntr_stats_t *stats, uint64_t cntr_value)
{
	cu_cpu_info_t	*cu_cpu_info = CPU->cpu_cu_info;
	uint_t		flags;
	uint64_t	delta;
	hrtime_t	time_delta;
	hrtime_t	time_snap;

	if (stats == NULL)
		return (-1);

	/*
	 * Nothing to do if counters are not programmed. This should not happen,
	 * but we check just in case.
	 */
	ASSERT(cu_flags & CU_FLAG_ON);
	ASSERT(cu_cpu_info != NULL);
	if (!(cu_flags & CU_FLAG_ON) ||
	    (cu_cpu_info == NULL))
		return (-2);

	flags = cu_cpu_info->cu_flag;
	ASSERT(flags & CU_CPU_CNTRS_ON);
	if (!(flags & CU_CPU_CNTRS_ON))
		return (-2);

	/*
	 * Take snapshot of high resolution timer
	 */
	time_snap = gethrtime();

	/*
	 * CU counters have just been programmed. We cannot assume that the new
	 * cntr_value continues from where we left off, so use the cntr_value as
	 * the new initial value.
	 */
	if (flags & CU_CPU_CNTRS_OFF_ON)
		stats->cs_value_start = cntr_value;

	/*
	 * Calculate delta in counter values between start of sampling period
	 * and now
	 */
	delta = cntr_value - stats->cs_value_start;

	/*
	 * Calculate time between start of sampling period and now
	 */
	time_delta = stats->cs_time_start ?
	    time_snap - stats->cs_time_start :
	    0;
	stats->cs_time_start = time_snap;
	stats->cs_value_start = cntr_value;

	if (time_delta > 0) { /* wrap shouldn't happen */
		/*
		 * Update either running or stopped time based on the transition
		 * state
		 */
		if (flags & CU_CPU_CNTRS_OFF_ON)
			stats->cs_time_stopped += time_delta;
		else
			stats->cs_time_running += time_delta;
	}

	/*
	 * Update rest of counter statistics if counter value didn't wrap
	 */
	if (delta > 0) {
		/*
		 * Update utilization rate if the interval between samples is
		 * sufficient.
		 */
		ASSERT(cu_sample_interval_min > CU_SCALE);
		if (time_delta > cu_sample_interval_min)
			stats->cs_rate = CU_RATE(delta, time_delta);
		if (stats->cs_rate_max < stats->cs_rate)
			stats->cs_rate_max = stats->cs_rate;

		stats->cs_value_last = delta;
		stats->cs_value_total += delta;
	}

	return (0);
}

/*
 * Update CMT PG utilization data.
 *
 * This routine computes the running total utilization and times for the
 * specified PG by adding up the total utilization and counter running and
 * stopped times of all CPUs in the PG and calculates the utilization rate and
 * maximum rate for all CPUs in the PG.
 */
void
cu_pg_update(pghw_t *pg)
{
	pg_cpu_itr_t	cpu_iter;
	pghw_type_t	pg_hwtype;
	cpu_t		*cpu;
	pghw_util_t	*hw_util = &pg->pghw_stats;
	uint64_t	old_utilization = hw_util->pghw_util;
	hrtime_t	now;
	hrtime_t	time_delta;
	uint64_t	utilization_delta;

	ASSERT(MUTEX_HELD(&cpu_lock));

	now = gethrtime();

	pg_hwtype = pg->pghw_hw;

	/*
	 * Initialize running total utilization and times for PG to 0
	 */
	hw_util->pghw_util = 0;
	hw_util->pghw_time_running = 0;
	hw_util->pghw_time_stopped = 0;

	/*
	 * Iterate over all CPUs in the PG and aggregate utilization, running
	 * time and stopped time.
	 */
	PG_CPU_ITR_INIT(pg, cpu_iter);
	while ((cpu = pg_cpu_next(&cpu_iter)) != NULL) {
		cu_cpu_info_t	*cu_cpu_info = cpu->cpu_cu_info;
		cu_cntr_info_t	*cntr_info;
		cu_cntr_stats_t	*stats;

		if (cu_cpu_info == NULL)
			continue;

		/*
		 * Update utilization data for the CPU and then
		 * aggregate per CPU running totals for PG
		 */
		(void) cu_cpu_update(cpu, B_TRUE);
		cntr_info = cu_cpu_info->cu_cntr_info[pg_hwtype];

		if (cntr_info == NULL || (stats = cntr_info->ci_stats) == NULL)
			continue;

		hw_util->pghw_util += stats->cs_value_total;
		hw_util->pghw_time_running += stats->cs_time_running;
		hw_util->pghw_time_stopped += stats->cs_time_stopped;

		/*
		 * If counters are stopped now, the pg_time_stopped was last
		 * updated at cs_time_start time. Add the time passed since then
		 * to the stopped time.
		 */
		if (!(cu_cpu_info->cu_flag & CU_CPU_CNTRS_ON))
			hw_util->pghw_time_stopped +=
			    now - stats->cs_time_start;
	}

	/*
	 * Compute per PG instruction rate and maximum rate
	 */
	time_delta = now - hw_util->pghw_time_stamp;
	hw_util->pghw_time_stamp = now;

	if (old_utilization == 0)
		return;

	/*
	 * Calculate change in utilization over sampling period and set this to
	 * 0 if the delta would be 0 or negative which may happen if any CPUs go
	 * offline during the sampling period
	 */
	if (hw_util->pghw_util > old_utilization)
		utilization_delta = hw_util->pghw_util - old_utilization;
	else
		utilization_delta = 0;

	/*
	 * Update utilization rate if the interval between samples is
	 * sufficient.
	 */
	ASSERT(cu_sample_interval_min > CU_SCALE);
	if (time_delta > CU_SAMPLE_INTERVAL_MIN)
		hw_util->pghw_rate = CU_RATE(utilization_delta, time_delta);

	/*
	 * Update the maximum observed rate
	 */
	if (hw_util->pghw_rate_max < hw_util->pghw_rate)
		hw_util->pghw_rate_max = hw_util->pghw_rate;
}