xref: /onnv-gate/usr/src/uts/sun4/os/x_call.c (revision 11389:dd00b884e84f)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/systm.h>
#include <sys/archsystm.h>
#include <sys/machsystm.h>
#include <sys/cpuvar.h>
#include <sys/intreg.h>
#include <sys/x_call.h>
#include <sys/cmn_err.h>
#include <sys/membar.h>
#include <sys/disp.h>
#include <sys/debug.h>
#include <sys/privregs.h>
#include <sys/xc_impl.h>
#include <sys/ivintr.h>
#include <sys/dmv.h>
#include <sys/sysmacros.h>

#ifdef TRAPTRACE
uint_t x_dstat[NCPU][XC_LOOP_EXIT+1];
uint_t x_rstat[NCPU][4];
#endif /* TRAPTRACE */

static uint64_t xc_serv_inum;	/* software interrupt number for xc_serv() */
static uint64_t xc_loop_inum;	/* software interrupt number for xc_loop() */
kmutex_t xc_sys_mutex;		/* protect xcall session and xc_mbox */
int xc_spl_enter[NCPU];		/* protect sending x-call */
static int xc_holder = -1; /* the cpu who initiates xc_attention, 0 is valid */

/*
 * Mail box for handshaking and xcall request; protected by xc_sys_mutex
 */
static struct xc_mbox {
	xcfunc_t *xc_func;
	uint64_t xc_arg1;
	uint64_t xc_arg2;
	cpuset_t xc_cpuset;
	volatile uint_t	xc_state;
} xc_mbox[NCPU];

uint64_t xc_tick_limit;		/* send_mondo() tick limit value */
uint64_t xc_tick_limit_scale = 1;	/* scale used to increase the limit */
uint64_t xc_tick_jump_limit;	/* send_mondo() irregular tick jump limit */
uint64_t xc_sync_tick_limit;	/* timeout limit for xt_sync() calls */

/* timeout value for xcalls to be received by the target CPU */
uint64_t xc_mondo_time_limit;

/* timeout value for xcall functions to be executed on the target CPU */
uint64_t xc_func_time_limit;

uint64_t xc_scale = 1;	/* scale used to calculate timeout limits */
uint64_t xc_mondo_multiplier = 10;

uint_t sendmondo_in_recover;

/*
 * sending x-calls
 */
void	init_mondo(xcfunc_t *func, uint64_t arg1, uint64_t arg2);
void	send_one_mondo(int cpuid);
void	send_mondo_set(cpuset_t set);

/*
 * Adjust xc_attention timeout if a faster cpu is dynamically added.
 * Ignore the dynamic removal of a cpu that would lower these timeout
 * values.
 */
static int
xc_func_timeout_adj(cpu_setup_t what, int cpuid) {
	uint64_t freq = cpunodes[cpuid].clock_freq;

	switch (what) {
	case CPU_ON:
	case CPU_INIT:
	case CPU_CONFIG:
	case CPU_CPUPART_IN:
		if (freq * xc_scale > xc_mondo_time_limit) {
			xc_mondo_time_limit = freq * xc_scale;
			xc_func_time_limit = xc_mondo_time_limit *
			    xc_mondo_multiplier;
		}
		break;
	case CPU_OFF:
	case CPU_UNCONFIG:
	case CPU_CPUPART_OUT:
	default:
		break;
	}

	return (0);
}

/*
 * xc_init - initialize x-call related locks
 */
void
xc_init(void)
{
	int pix;
	uint64_t maxfreq = 0;

	mutex_init(&xc_sys_mutex, NULL, MUTEX_SPIN,
	    (void *)ipltospl(XCALL_PIL));

#ifdef TRAPTRACE
	/* Initialize for all possible CPUs. */
	for (pix = 0; pix < NCPU; pix++) {
		XC_STAT_INIT(pix);
	}
#endif /* TRAPTRACE */

	xc_serv_inum = add_softintr(XCALL_PIL, (softintrfunc)xc_serv, 0,
	    SOFTINT_MT);
	xc_loop_inum = add_softintr(XCALL_PIL, (softintrfunc)xc_loop, 0,
	    SOFTINT_MT);

	/*
	 * Initialize the calibrated tick limit for send_mondo.
	 * The value represents the maximum tick count to wait.
	 */
	xc_tick_limit =
	    ((uint64_t)sys_tick_freq * XC_SEND_MONDO_MSEC) / 1000;
	xc_tick_jump_limit = xc_tick_limit / 32;
	xc_tick_limit *= xc_tick_limit_scale;
	xc_sync_tick_limit = xc_tick_limit;

	/*
	 * Maximum number of loops to wait before timing out in xc_attention.
	 */
	for (pix = 0; pix < NCPU; pix++) {
		maxfreq = MAX(cpunodes[pix].clock_freq, maxfreq);
	}
	xc_mondo_time_limit = maxfreq * xc_scale;
	register_cpu_setup_func((cpu_setup_func_t *)xc_func_timeout_adj, NULL);

	/*
	 * Maximum number of loops to wait for a xcall function to be
	 * executed on the target CPU.
	 */
	xc_func_time_limit = xc_mondo_time_limit * xc_mondo_multiplier;
}

/*
 * The following routines basically provide callers with two kinds of
 * inter-processor interrupt services:
 *	1. cross calls (x-calls) - requests are handled at target cpu's TL=0
 *	2. cross traps (c-traps) - requests are handled at target cpu's TL>0
 *
 * Although these routines protect the services from migrating to other cpus
 * "after" they are called, it is the caller's choice or responsibility to
 * prevent the cpu migration "before" calling them.
 *
 * X-call routines:
 *
 *	xc_one()  - send a request to one processor
 *	xc_some() - send a request to some processors
 *	xc_all()  - send a request to all processors
 *
 *	Their common parameters:
 *		func - a TL=0 handler address
 *		arg1 and arg2  - optional
 *
 *	The services provided by x-call routines allow callers
 *	to send a request to target cpus to execute a TL=0
 *	handler.
 *	The interface of the registers of the TL=0 handler:
 *		%o0: arg1
 *		%o1: arg2
 *
 * X-trap routines:
 *
 *	xt_one()  - send a request to one processor
 *	xt_some() - send a request to some processors
 *	xt_all()  - send a request to all processors
 *
 *	Their common parameters:
 *		func - a TL>0 handler address or an interrupt number
 *		arg1, arg2
 *		       optional when "func" is an address;
 *		       0        when "func" is an interrupt number
 *
 *	If the request of "func" is a kernel address, then
 *	the target cpu will execute the request of "func" with
 *	args at "TL>0" level.
 *	The interface of the registers of the TL>0 handler:
 *		%g1: arg1
 *		%g2: arg2
 *
 *	If the request of "func" is not a kernel address, then it has
 *	to be an assigned interrupt number through add_softintr().
 *	An interrupt number is an index to the interrupt vector table,
 *	which entry contains an interrupt handler address with its
 *	corresponding interrupt level and argument.
 *	The target cpu will arrange the request to be serviced according
 *	to its pre-registered information.
 *	args are assumed to be zeros in this case.
 *
 * In addition, callers are allowed to capture and release cpus by
 * calling the routines: xc_attention() and xc_dismissed().
 */

/*
 * spl_xcall - set PIL to xcall level
 */
int
spl_xcall(void)
{
	return (splr(XCALL_PIL));
}

/*
 * xt_one - send a "x-trap" to a cpu
 */
void
xt_one(int cix, xcfunc_t *func, uint64_t arg1, uint64_t arg2)
{
	if (!CPU_IN_SET(cpu_ready_set, cix)) {
		return;
	}
	xt_one_unchecked(cix, func, arg1, arg2);
}

/*
 * xt_one_unchecked - send a "x-trap" to a cpu without checking for its
 * existance in cpu_ready_set
 */
void
xt_one_unchecked(int cix, xcfunc_t *func, uint64_t arg1, uint64_t arg2)
{
	int lcx;
	int opl;
	cpuset_t tset;

	/*
	 * Make sure the function address will not be interpreted as a
	 * dmv interrupt
	 */
	ASSERT(!DMV_IS_DMV(func));

	/*
	 * It's illegal to send software inums through the cross-trap
	 * interface.
	 */
	ASSERT((uintptr_t)func >= KERNELBASE);

	CPUSET_ZERO(tset);

	XC_SPL_ENTER(lcx, opl);			/* lcx set by the macro */

	CPUSET_ADD(tset, cix);

	if (cix == lcx) {
		/*
		 * same cpu - use software fast trap
		 */
		send_self_xcall(CPU, arg1, arg2, func);
		XC_STAT_INC(x_dstat[lcx][XT_ONE_SELF]);
		XC_TRACE(XT_ONE_SELF, &tset, func, arg1, arg2);
	} else {	/* other cpu - send a mondo to the target cpu */
		/*
		 * other cpu - send a mondo to the target cpu
		 */
		XC_TRACE(XT_ONE_OTHER, &tset, func, arg1, arg2);
		init_mondo(func, arg1, arg2);
		send_one_mondo(cix);
		XC_STAT_INC(x_dstat[lcx][XT_ONE_OTHER]);
	}
	XC_SPL_EXIT(lcx, opl);
}

/*
 * xt_some - send a "x-trap" to some cpus
 */
void
xt_some(cpuset_t cpuset, xcfunc_t *func, uint64_t arg1, uint64_t arg2)
{
	int lcx;
	int opl;
	cpuset_t xc_cpuset, tset;

	/*
	 * Make sure the function address will not be interpreted as a
	 * dmv interrupt
	 */
	ASSERT(!DMV_IS_DMV(func));

	/*
	 * It's illegal to send software inums through the cross-trap
	 * interface.
	 */
	ASSERT((uintptr_t)func >= KERNELBASE);

	CPUSET_ZERO(tset);

	XC_SPL_ENTER(lcx, opl);		/* lcx set by the macro */

	CPUSET_ADD(tset, lcx);

	/*
	 * only send to the CPU_READY ones
	 */
	xc_cpuset = cpu_ready_set;
	CPUSET_AND(xc_cpuset, cpuset);

	/*
	 * send to nobody; just return
	 */
	if (CPUSET_ISNULL(xc_cpuset)) {
		XC_SPL_EXIT(lcx, opl);
		return;
	}

	/*
	 * don't send mondo to self
	 */
	if (CPU_IN_SET(xc_cpuset, lcx)) {
		/*
		 * same cpu - use software fast trap
		 */
		send_self_xcall(CPU, arg1, arg2, func);
		XC_STAT_INC(x_dstat[lcx][XT_SOME_SELF]);
		XC_TRACE(XT_SOME_SELF, &tset, func, arg1, arg2);
		CPUSET_DEL(xc_cpuset, lcx);
		if (CPUSET_ISNULL(xc_cpuset)) {
			XC_SPL_EXIT(lcx, opl);
			return;
		}
	}
	XC_TRACE(XT_SOME_OTHER, &xc_cpuset, func, arg1, arg2);
	init_mondo(func, arg1, arg2);
	send_mondo_set(xc_cpuset);
	XC_STAT_INC(x_dstat[lcx][XT_SOME_OTHER]);

	XC_SPL_EXIT(lcx, opl);
}

/*
 * xt_all - send a "x-trap" to all cpus
 */
void
xt_all(xcfunc_t *func, uint64_t arg1, uint64_t arg2)
{
	int lcx;
	int opl;
	cpuset_t xc_cpuset, tset;

	/*
	 * Make sure the function address will not be interpreted as a
	 * dmv interrupt
	 */
	ASSERT(!DMV_IS_DMV(func));

	/*
	 * It's illegal to send software inums through the cross-trap
	 * interface.
	 */
	ASSERT((uintptr_t)func >= KERNELBASE);

	CPUSET_ZERO(tset);

	XC_SPL_ENTER(lcx, opl);		/* lcx set by the macro */

	CPUSET_ADD(tset, lcx);

	/*
	 * same cpu - use software fast trap
	 */
	if (CPU_IN_SET(cpu_ready_set, lcx))
		send_self_xcall(CPU, arg1, arg2, func);

	XC_TRACE(XT_ALL_OTHER, &cpu_ready_set, func, arg1, arg2);

	/*
	 * don't send mondo to self
	 */
	xc_cpuset = cpu_ready_set;
	CPUSET_DEL(xc_cpuset, lcx);

	if (CPUSET_ISNULL(xc_cpuset)) {
		XC_STAT_INC(x_dstat[lcx][XT_ALL_SELF]);
		XC_TRACE(XT_ALL_SELF, &tset, func, arg1, arg2);
		XC_SPL_EXIT(lcx, opl);
		return;
	}

	init_mondo(func, arg1, arg2);
	send_mondo_set(xc_cpuset);

	XC_STAT_INC(x_dstat[lcx][XT_ALL_OTHER]);
	XC_SPL_EXIT(lcx, opl);
}

/*
 * xc_one - send a "x-call" to a cpu
 */
void
xc_one(int cix, xcfunc_t *func, uint64_t arg1, uint64_t arg2)
{
	int lcx;
	int opl;
	uint64_t loop_cnt = 0;
	cpuset_t tset;
	int first_time = 1;

	/*
	 * send to nobody; just return
	 */
	if (!CPU_IN_SET(cpu_ready_set, cix))
		return;

	ASSERT((uintptr_t)func > KERNELBASE);
	ASSERT(((uintptr_t)func % PC_ALIGN) == 0);

	CPUSET_ZERO(tset);

	kpreempt_disable();

	XC_SPL_ENTER(lcx, opl);		/* lcx set by the macro */

	CPUSET_ADD(tset, cix);

	if (cix == lcx) {	/* same cpu just do it */
		XC_TRACE(XC_ONE_SELF, &tset, func, arg1, arg2);
		(*func)(arg1, arg2);
		XC_STAT_INC(x_dstat[lcx][XC_ONE_SELF]);
		XC_SPL_EXIT(lcx, opl);
		kpreempt_enable();
		return;
	}

	if (xc_holder == lcx) {		/* got the xc_sys_mutex already */
		ASSERT(MUTEX_HELD(&xc_sys_mutex));
		ASSERT(CPU_IN_SET(xc_mbox[lcx].xc_cpuset, lcx));
		ASSERT(CPU_IN_SET(xc_mbox[cix].xc_cpuset, cix));
		ASSERT(xc_mbox[cix].xc_state == XC_WAIT);
		XC_TRACE(XC_ONE_OTHER_H, &tset, func, arg1, arg2);

		/*
		 * target processor's xc_loop should be waiting
		 * for the work to do; just set up the xc_mbox
		 */
		XC_SETUP(cix, func, arg1, arg2);
		membar_stld();

		while (xc_mbox[cix].xc_state != XC_WAIT) {
			if (loop_cnt++ > xc_func_time_limit) {
				if (sendmondo_in_recover) {
					drv_usecwait(1);
					loop_cnt = 0;
					continue;
				}
				cmn_err(CE_PANIC, "xc_one() timeout, "
				    "xc_state[%d] != XC_WAIT", cix);
			}
		}
		XC_STAT_INC(x_dstat[lcx][XC_ONE_OTHER_H]);
		XC_SPL_EXIT(lcx, opl);
		kpreempt_enable();
		return;
	}

	/*
	 * Avoid dead lock if someone has sent us a xc_loop request while
	 * we are trying to grab xc_sys_mutex.
	 */
	XC_SPL_EXIT(lcx, opl);

	/*
	 * At this point, since we don't own xc_sys_mutex,
	 * our pil shouldn't run at or above the XCALL_PIL.
	 */
	ASSERT(getpil() < XCALL_PIL);

	/*
	 * Since xc_holder is not owned by us, it could be that
	 * no one owns it, or we are not informed to enter into
	 * xc_loop(). In either case, we need to grab the
	 * xc_sys_mutex before we write to the xc_mbox, and
	 * we shouldn't release it until the request is finished.
	 */

	mutex_enter(&xc_sys_mutex);
	xc_spl_enter[lcx] = 1;

	/*
	 * Since we own xc_sys_mutex now, we are safe to
	 * write to the xc_mbox.
	 */
	ASSERT(xc_mbox[cix].xc_state == XC_IDLE);
	XC_TRACE(XC_ONE_OTHER, &tset, func, arg1, arg2);
	XC_SETUP(cix, func, arg1, arg2);
	init_mondo(setsoftint_tl1, xc_serv_inum, 0);
	send_one_mondo(cix);
	xc_spl_enter[lcx] = 0;

	/* xc_serv does membar_stld */
	while (xc_mbox[cix].xc_state != XC_IDLE) {
		if (loop_cnt++ > xc_func_time_limit) {
			if (sendmondo_in_recover) {
				drv_usecwait(1);
				loop_cnt = 0;
				continue;
			}
			if (first_time) {
				XT_SYNC_ONE(cix);
				first_time = 0;
				loop_cnt = 0;
				continue;
			}
			cmn_err(CE_PANIC, "xc_one() timeout, "
			    "xc_state[%d] != XC_IDLE", cix);
		}
	}
	XC_STAT_INC(x_dstat[lcx][XC_ONE_OTHER]);
	mutex_exit(&xc_sys_mutex);

	kpreempt_enable();
}

/*
 * xc_some - send a "x-call" to some cpus; sending to self is excluded
 */
void
xc_some(cpuset_t cpuset, xcfunc_t *func, uint64_t arg1, uint64_t arg2)
{
	int lcx;
	int opl;
	cpuset_t xc_cpuset, tset;

	ASSERT((uintptr_t)func > KERNELBASE);
	ASSERT(((uintptr_t)func % PC_ALIGN) == 0);

	CPUSET_ZERO(tset);

	kpreempt_disable();
	XC_SPL_ENTER(lcx, opl);			/* lcx set by the macro */

	CPUSET_ADD(tset, lcx);

	/*
	 * only send to the CPU_READY ones
	 */
	xc_cpuset = cpu_ready_set;
	CPUSET_AND(xc_cpuset, cpuset);

	/*
	 * send to nobody; just return
	 */
	if (CPUSET_ISNULL(xc_cpuset)) {
		XC_SPL_EXIT(lcx, opl);
		kpreempt_enable();
		return;
	}

	if (CPU_IN_SET(xc_cpuset, lcx)) {
		/*
		 * same cpu just do it
		 */
		(*func)(arg1, arg2);
		CPUSET_DEL(xc_cpuset, lcx);
		if (CPUSET_ISNULL(xc_cpuset)) {
			XC_STAT_INC(x_dstat[lcx][XC_SOME_SELF]);
			XC_TRACE(XC_SOME_SELF, &tset, func, arg1, arg2);
			XC_SPL_EXIT(lcx, opl);
			kpreempt_enable();
			return;
		}
	}

	if (xc_holder == lcx) {		/* got the xc_sys_mutex already */
		cpuset_t mset = xc_mbox[lcx].xc_cpuset;

		CPUSET_AND(mset, cpuset);
		ASSERT(MUTEX_HELD(&xc_sys_mutex));
		ASSERT(CPUSET_ISEQUAL(mset, cpuset));
		SEND_MBOX_ONLY(xc_cpuset, func, arg1, arg2, lcx, XC_WAIT);
		WAIT_MBOX_DONE(xc_cpuset, lcx, XC_WAIT, 0);
		XC_STAT_INC(x_dstat[lcx][XC_SOME_OTHER_H]);
		XC_TRACE(XC_SOME_OTHER_H, &xc_cpuset, func, arg1, arg2);
		XC_SPL_EXIT(lcx, opl);
		kpreempt_enable();
		return;
	}

	/*
	 * Avoid dead lock if someone has sent us a xc_loop request while
	 * we are trying to grab xc_sys_mutex.
	 */
	XC_SPL_EXIT(lcx, opl);

	/*
	 * At this point, since we don't own xc_sys_mutex,
	 * our pil shouldn't run at or above the XCALL_PIL.
	 */
	ASSERT(getpil() < XCALL_PIL);

	/*
	 * grab xc_sys_mutex before writing to the xc_mbox
	 */
	mutex_enter(&xc_sys_mutex);
	xc_spl_enter[lcx] = 1;

	XC_TRACE(XC_SOME_OTHER, &xc_cpuset, func, arg1, arg2);
	init_mondo(setsoftint_tl1, xc_serv_inum, 0);
	SEND_MBOX_MONDO(xc_cpuset, func, arg1, arg2, XC_IDLE);
	WAIT_MBOX_DONE(xc_cpuset, lcx, XC_IDLE, 1);

	xc_spl_enter[lcx] = 0;
	XC_STAT_INC(x_dstat[lcx][XC_SOME_OTHER]);
	mutex_exit(&xc_sys_mutex);
	kpreempt_enable();
}

/*
 * xc_all - send a "x-call" to all cpus
 */
void
xc_all(xcfunc_t *func, uint64_t arg1, uint64_t arg2)
{
	int lcx;
	int opl;
	cpuset_t xc_cpuset, tset;

	ASSERT((uintptr_t)func > KERNELBASE);
	ASSERT(((uintptr_t)func % PC_ALIGN) == 0);

	CPUSET_ZERO(tset);

	kpreempt_disable();
	XC_SPL_ENTER(lcx, opl);			/* lcx set by the macro */

	CPUSET_ADD(tset, lcx);

	/*
	 * same cpu just do it
	 */
	(*func)(arg1, arg2);
	xc_cpuset = cpu_ready_set;
	CPUSET_DEL(xc_cpuset, lcx);

	if (CPUSET_ISNULL(xc_cpuset)) {
		XC_STAT_INC(x_dstat[lcx][XC_ALL_SELF]);
		XC_TRACE(XC_ALL_SELF, &tset, func, arg1, arg2);
		XC_SPL_EXIT(lcx, opl);
		kpreempt_enable();
		return;
	}

	if (xc_holder == lcx) {		/* got the xc_sys_mutex already */
		cpuset_t mset = xc_mbox[lcx].xc_cpuset;

		CPUSET_AND(mset, xc_cpuset);
		ASSERT(MUTEX_HELD(&xc_sys_mutex));
		ASSERT(CPUSET_ISEQUAL(mset, xc_cpuset));
		XC_TRACE(XC_ALL_OTHER_H, &xc_cpuset, func, arg1, arg2);
		SEND_MBOX_ONLY(xc_cpuset, func, arg1, arg2, lcx, XC_WAIT);
		WAIT_MBOX_DONE(xc_cpuset, lcx, XC_WAIT, 0);
		XC_STAT_INC(x_dstat[lcx][XC_ALL_OTHER_H]);
		XC_SPL_EXIT(lcx, opl);
		kpreempt_enable();
		return;
	}

	/*
	 * Avoid dead lock if someone has sent us a xc_loop request while
	 * we are trying to grab xc_sys_mutex.
	 */
	XC_SPL_EXIT(lcx, opl);

	/*
	 * At this point, since we don't own xc_sys_mutex,
	 * our pil shouldn't run at or above the XCALL_PIL.
	 */
	ASSERT(getpil() < XCALL_PIL);

	/*
	 * grab xc_sys_mutex before writing to the xc_mbox
	 */
	mutex_enter(&xc_sys_mutex);
	xc_spl_enter[lcx] = 1;

	XC_TRACE(XC_ALL_OTHER, &xc_cpuset, func, arg1, arg2);
	init_mondo(setsoftint_tl1, xc_serv_inum, 0);
	SEND_MBOX_MONDO(xc_cpuset, func, arg1, arg2, XC_IDLE);
	WAIT_MBOX_DONE(xc_cpuset, lcx, XC_IDLE, 1);

	xc_spl_enter[lcx] = 0;
	XC_STAT_INC(x_dstat[lcx][XC_ALL_OTHER]);
	mutex_exit(&xc_sys_mutex);
	kpreempt_enable();
}

/*
 * xc_attention - paired with xc_dismissed()
 *
 * xt_attention() holds the xc_sys_mutex and xc_dismissed() releases it
 * called when an initiator wants to capture some/all cpus for a critical
 * session.
 */
void
xc_attention(cpuset_t cpuset)
{
	int pix, lcx;
	cpuset_t xc_cpuset, tmpset;
	cpuset_t recv_cpuset;
	uint64_t loop_cnt = 0;
	int first_time = 1;

	CPUSET_ZERO(recv_cpuset);

	/*
	 * don't migrate the cpu until xc_dismissed() is finished
	 */
	ASSERT(getpil() < XCALL_PIL);
	mutex_enter(&xc_sys_mutex);
	lcx = (int)(CPU->cpu_id);
	ASSERT(x_dstat[lcx][XC_ATTENTION] ==
	    x_dstat[lcx][XC_DISMISSED]);
	ASSERT(xc_holder == -1);
	xc_mbox[lcx].xc_cpuset = cpuset;
	xc_holder = lcx; /* no membar; only current cpu needs the right lcx */

	/*
	 * only send to the CPU_READY ones
	 */
	xc_cpuset = cpu_ready_set;
	CPUSET_AND(xc_cpuset, cpuset);

	/*
	 * don't send mondo to self
	 */
	CPUSET_DEL(xc_cpuset, lcx);

	XC_STAT_INC(x_dstat[lcx][XC_ATTENTION]);
	XC_TRACE(XC_ATTENTION, &xc_cpuset, NULL, NULL, NULL);

	if (CPUSET_ISNULL(xc_cpuset))
		return;

	xc_spl_enter[lcx] = 1;
	/*
	 * inform the target processors to enter into xc_loop()
	 */
	init_mondo(setsoftint_tl1, xc_loop_inum, 0);
	SEND_MBOX_MONDO_XC_ENTER(xc_cpuset);
	xc_spl_enter[lcx] = 0;

	/*
	 * make sure target processors have entered into xc_loop()
	 */
	while (!CPUSET_ISEQUAL(recv_cpuset, xc_cpuset)) {
		tmpset = xc_cpuset;
		for (pix = 0; pix < NCPU; pix++) {
			if (CPU_IN_SET(tmpset, pix)) {
				/*
				 * membar_stld() is done in xc_loop
				 */
				if (xc_mbox[pix].xc_state == XC_WAIT) {
					CPUSET_ADD(recv_cpuset, pix);
				}
				CPUSET_DEL(tmpset, pix);
				if (CPUSET_ISNULL(tmpset)) {
					break;
				}
			}
		}
		if (loop_cnt++ > xc_mondo_time_limit) {
			if (sendmondo_in_recover) {
				drv_usecwait(1);
				loop_cnt = 0;
				continue;
			}
			if (first_time) {
				XT_SYNC_SOME(xc_cpuset);
				first_time = 0;
				loop_cnt = 0;
				continue;
			}
			cmn_err(CE_PANIC, "xc_attention() timeout");
		}
	}

	/*
	 * xc_sys_mutex remains held until xc_dismissed() is finished
	 */
}

/*
 * xc_dismissed - paired with xc_attention()
 *
 * Called after the critical session is finished.
 */
void
xc_dismissed(cpuset_t cpuset)
{
	int pix;
	int lcx = (int)(CPU->cpu_id);
	cpuset_t xc_cpuset, tmpset;
	cpuset_t recv_cpuset;
	uint64_t loop_cnt = 0;

	ASSERT(lcx == xc_holder);
	ASSERT(CPUSET_ISEQUAL(xc_mbox[lcx].xc_cpuset, cpuset));
	ASSERT(getpil() >= XCALL_PIL);
	CPUSET_ZERO(xc_mbox[lcx].xc_cpuset);
	CPUSET_ZERO(recv_cpuset);
	membar_stld();

	XC_STAT_INC(x_dstat[lcx][XC_DISMISSED]);
	ASSERT(x_dstat[lcx][XC_DISMISSED] == x_dstat[lcx][XC_ATTENTION]);

	/*
	 * only send to the CPU_READY ones
	 */
	xc_cpuset = cpu_ready_set;
	CPUSET_AND(xc_cpuset, cpuset);

	/*
	 * exclude itself
	 */
	CPUSET_DEL(xc_cpuset, lcx);
	XC_TRACE(XC_DISMISSED, &xc_cpuset, NULL, NULL, NULL);
	if (CPUSET_ISNULL(xc_cpuset)) {
		xc_holder = -1;
		mutex_exit(&xc_sys_mutex);
		return;
	}

	/*
	 * inform other processors to get out of xc_loop()
	 */
	tmpset = xc_cpuset;
	for (pix = 0; pix < NCPU; pix++) {
		if (CPU_IN_SET(tmpset, pix)) {
			xc_mbox[pix].xc_state = XC_EXIT;
			membar_stld();
			CPUSET_DEL(tmpset, pix);
			if (CPUSET_ISNULL(tmpset)) {
				break;
			}
		}
	}

	/*
	 * make sure target processors have exited from xc_loop()
	 */
	while (!CPUSET_ISEQUAL(recv_cpuset, xc_cpuset)) {
		tmpset = xc_cpuset;
		for (pix = 0; pix < NCPU; pix++) {
			if (CPU_IN_SET(tmpset, pix)) {
				/*
				 * membar_stld() is done in xc_loop
				 */
				if (xc_mbox[pix].xc_state == XC_IDLE) {
					CPUSET_ADD(recv_cpuset, pix);
				}
				CPUSET_DEL(tmpset, pix);
				if (CPUSET_ISNULL(tmpset)) {
					break;
				}
			}
		}
		if (loop_cnt++ > xc_func_time_limit) {
				if (sendmondo_in_recover) {
					drv_usecwait(1);
					loop_cnt = 0;
					continue;
				}
			cmn_err(CE_PANIC, "xc_dismissed() timeout");
		}
	}
	xc_holder = -1;
	mutex_exit(&xc_sys_mutex);
}

/*
 * xc_serv - "x-call" handler at TL=0; serves only one x-call request
 * runs at XCALL_PIL level.
 */
uint_t
xc_serv(void)
{
	int lcx = (int)(CPU->cpu_id);
	struct xc_mbox *xmp;
	xcfunc_t *func;
	uint64_t arg1, arg2;
	cpuset_t tset;

	ASSERT(getpil() == XCALL_PIL);
	CPUSET_ZERO(tset);
	CPUSET_ADD(tset, lcx);
	flush_windows();
	xmp = &xc_mbox[lcx];
	ASSERT(lcx != xc_holder);
	ASSERT(xmp->xc_state == XC_DOIT);
	func = xmp->xc_func;
	XC_TRACE(XC_SERV, &tset, func, xmp->xc_arg1, xmp->xc_arg2);
	if (func != NULL) {
		arg1 = xmp->xc_arg1;
		arg2 = xmp->xc_arg2;
		(*func)(arg1, arg2);
	}
	XC_STAT_INC(x_rstat[lcx][XC_SERV]);
	XC_TRACE(XC_SERV, &tset, func, arg1, arg2);
	xmp->xc_state = XC_IDLE;
	membar_stld();
	return (1);
}

/*
 * if == 1, an xc_loop timeout will cause a panic
 * otherwise print a warning
 */
uint_t xc_loop_panic = 0;

/*
 * xc_loop - "x-call" handler at TL=0; capture the cpu for a critial
 * session, or serve multiple x-call requests runs at XCALL_PIL level.
 */
uint_t
xc_loop(void)
{
	int lcx = (int)(CPU->cpu_id);
	struct xc_mbox *xmp;
	xcfunc_t *func;
	uint64_t arg1, arg2;
	uint64_t loop_cnt = 0;
	cpuset_t tset;

	ASSERT(getpil() == XCALL_PIL);

	CPUSET_ZERO(tset);
	flush_windows();

	/*
	 * Some one must have owned the xc_sys_mutex;
	 * no further interrupt (at XCALL_PIL or below) can
	 * be taken by this processor until xc_loop exits.
	 *
	 * The owner of xc_sys_mutex (or xc_holder) can expect
	 * its xc/xt requests are handled as follows:
	 * 	xc requests use xc_mbox's handshaking for their services
	 * 	xt requests at TL>0 will be handled immediately
	 * 	xt requests at TL=0:
	 *		if their handlers'pils are <= XCALL_PIL, then
	 *			they will be handled after xc_loop exits
	 *			(so, they probably should not be used)
	 *		else they will be handled immediately
	 *
	 * For those who are not informed to enter xc_loop, if they
	 * send xc/xt requests to this processor at this moment,
	 * the requests will be handled as follows:
	 *	xc requests will be handled after they grab xc_sys_mutex
	 *	xt requests at TL>0 will be handled immediately
	 * 	xt requests at TL=0:
	 *		if their handlers'pils are <= XCALL_PIL, then
	 *			they will be handled after xc_loop exits
	 *		else they will be handled immediately
	 */
	xmp = &xc_mbox[lcx];
	ASSERT(lcx != xc_holder);
	ASSERT(xmp->xc_state == XC_ENTER);
	xmp->xc_state = XC_WAIT;
	CPUSET_ADD(tset, lcx);
	membar_stld();
	XC_STAT_INC(x_rstat[lcx][XC_LOOP]);
	XC_TRACE(XC_LOOP_ENTER, &tset, NULL, NULL, NULL);
	while (xmp->xc_state != XC_EXIT) {
		if (xmp->xc_state == XC_DOIT) {
			func = xmp->xc_func;
			arg1 = xmp->xc_arg1;
			arg2 = xmp->xc_arg2;
			XC_TRACE(XC_LOOP_DOIT, &tset, func, arg1, arg2);
			if (func != NULL)
				(*func)(arg1, arg2);
			xmp->xc_state = XC_WAIT;
			membar_stld();
			/*
			 * reset the timeout counter
			 * since some work was done
			 */
			loop_cnt = 0;
		} else {
			/* patience is a virtue... */
			loop_cnt++;
		}

		if (loop_cnt > xc_func_time_limit) {
			if (sendmondo_in_recover) {
				drv_usecwait(1);
				loop_cnt = 0;
				continue;
			}
			cmn_err(xc_loop_panic ? CE_PANIC : CE_WARN,
			    "xc_loop() timeout");
			/*
			 * if the above displayed a warning,
			 * reset the timeout counter and be patient
			 */
			loop_cnt = 0;
		}
	}
	ASSERT(xmp->xc_state == XC_EXIT);
	ASSERT(xc_holder != -1);
	XC_TRACE(XC_LOOP_EXIT, &tset, NULL, NULL, NULL);
	xmp->xc_state = XC_IDLE;
	membar_stld();
	return (1);
}