1*0Sstevel@tonic-gate /* 2*0Sstevel@tonic-gate * CDDL HEADER START 3*0Sstevel@tonic-gate * 4*0Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*0Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*0Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*0Sstevel@tonic-gate * with the License. 8*0Sstevel@tonic-gate * 9*0Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*0Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*0Sstevel@tonic-gate * See the License for the specific language governing permissions 12*0Sstevel@tonic-gate * and limitations under the License. 13*0Sstevel@tonic-gate * 14*0Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*0Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*0Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*0Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*0Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*0Sstevel@tonic-gate * 20*0Sstevel@tonic-gate * CDDL HEADER END 21*0Sstevel@tonic-gate */ 22*0Sstevel@tonic-gate /* 23*0Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24*0Sstevel@tonic-gate * Use is subject to license terms. 25*0Sstevel@tonic-gate */ 26*0Sstevel@tonic-gate 27*0Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 28*0Sstevel@tonic-gate 29*0Sstevel@tonic-gate /* 30*0Sstevel@tonic-gate * Facilities for cross-processor subroutine calls using "mailbox" interrupts. 31*0Sstevel@tonic-gate * 32*0Sstevel@tonic-gate */ 33*0Sstevel@tonic-gate 34*0Sstevel@tonic-gate #include <sys/types.h> 35*0Sstevel@tonic-gate 36*0Sstevel@tonic-gate #include <sys/param.h> 37*0Sstevel@tonic-gate #include <sys/t_lock.h> 38*0Sstevel@tonic-gate #include <sys/thread.h> 39*0Sstevel@tonic-gate #include <sys/cpuvar.h> 40*0Sstevel@tonic-gate #include <sys/x_call.h> 41*0Sstevel@tonic-gate #include <sys/cpu.h> 42*0Sstevel@tonic-gate #include <sys/psw.h> 43*0Sstevel@tonic-gate #include <sys/sunddi.h> 44*0Sstevel@tonic-gate #include <sys/mmu.h> 45*0Sstevel@tonic-gate #include <sys/debug.h> 46*0Sstevel@tonic-gate #include <sys/systm.h> 47*0Sstevel@tonic-gate #include <sys/machsystm.h> 48*0Sstevel@tonic-gate #include <sys/mutex_impl.h> 49*0Sstevel@tonic-gate 50*0Sstevel@tonic-gate static struct xc_mbox xc_mboxes[X_CALL_LEVELS]; 51*0Sstevel@tonic-gate static kmutex_t xc_mbox_lock[X_CALL_LEVELS]; 52*0Sstevel@tonic-gate static uint_t xc_xlat_xcptoipl[X_CALL_LEVELS] = { 53*0Sstevel@tonic-gate XC_LO_PIL, 54*0Sstevel@tonic-gate XC_MED_PIL, 55*0Sstevel@tonic-gate XC_HI_PIL 56*0Sstevel@tonic-gate }; 57*0Sstevel@tonic-gate 58*0Sstevel@tonic-gate static void xc_common(xc_func_t, xc_arg_t, xc_arg_t, xc_arg_t, 59*0Sstevel@tonic-gate int, cpuset_t, int); 60*0Sstevel@tonic-gate 61*0Sstevel@tonic-gate static int xc_initialized = 0; 62*0Sstevel@tonic-gate extern ulong_t cpu_ready_set; 63*0Sstevel@tonic-gate 64*0Sstevel@tonic-gate void 65*0Sstevel@tonic-gate xc_init() 66*0Sstevel@tonic-gate { 67*0Sstevel@tonic-gate /* 68*0Sstevel@tonic-gate * By making these mutexes type MUTEX_DRIVER, the ones below 69*0Sstevel@tonic-gate * LOCK_LEVEL will be implemented as adaptive mutexes, and the 70*0Sstevel@tonic-gate * ones above LOCK_LEVEL will be spin mutexes. 71*0Sstevel@tonic-gate */ 72*0Sstevel@tonic-gate mutex_init(&xc_mbox_lock[0], NULL, MUTEX_DRIVER, 73*0Sstevel@tonic-gate (void *)ipltospl(XC_LO_PIL)); 74*0Sstevel@tonic-gate mutex_init(&xc_mbox_lock[1], NULL, MUTEX_DRIVER, 75*0Sstevel@tonic-gate (void *)ipltospl(XC_MED_PIL)); 76*0Sstevel@tonic-gate mutex_init(&xc_mbox_lock[2], NULL, MUTEX_DRIVER, 77*0Sstevel@tonic-gate (void *)ipltospl(XC_HI_PIL)); 78*0Sstevel@tonic-gate 79*0Sstevel@tonic-gate xc_initialized = 1; 80*0Sstevel@tonic-gate } 81*0Sstevel@tonic-gate 82*0Sstevel@tonic-gate /* 83*0Sstevel@tonic-gate * Used by the debugger to determine whether or not cross calls have been 84*0Sstevel@tonic-gate * initialized and are safe to use. 85*0Sstevel@tonic-gate */ 86*0Sstevel@tonic-gate int 87*0Sstevel@tonic-gate kdi_xc_initialized(void) 88*0Sstevel@tonic-gate { 89*0Sstevel@tonic-gate return (xc_initialized); 90*0Sstevel@tonic-gate } 91*0Sstevel@tonic-gate 92*0Sstevel@tonic-gate #define CAPTURE_CPU_ARG 0xffffffff 93*0Sstevel@tonic-gate 94*0Sstevel@tonic-gate /* 95*0Sstevel@tonic-gate * X-call interrupt service routine. 96*0Sstevel@tonic-gate * 97*0Sstevel@tonic-gate * arg == X_CALL_MEDPRI - capture cpus. 98*0Sstevel@tonic-gate * 99*0Sstevel@tonic-gate * We're protected against changing CPUs by being a high-priority interrupt. 100*0Sstevel@tonic-gate */ 101*0Sstevel@tonic-gate /*ARGSUSED*/ 102*0Sstevel@tonic-gate uint_t 103*0Sstevel@tonic-gate xc_serv(caddr_t arg1, caddr_t arg2) 104*0Sstevel@tonic-gate { 105*0Sstevel@tonic-gate int op; 106*0Sstevel@tonic-gate int pri = (int)(uintptr_t)arg1; 107*0Sstevel@tonic-gate struct cpu *cpup = CPU; 108*0Sstevel@tonic-gate xc_arg_t *argp; 109*0Sstevel@tonic-gate xc_arg_t arg2val; 110*0Sstevel@tonic-gate uint_t tlbflush; 111*0Sstevel@tonic-gate 112*0Sstevel@tonic-gate if (pri == X_CALL_MEDPRI) { 113*0Sstevel@tonic-gate 114*0Sstevel@tonic-gate argp = &xc_mboxes[X_CALL_MEDPRI].arg2; 115*0Sstevel@tonic-gate arg2val = *argp; 116*0Sstevel@tonic-gate if (arg2val != CAPTURE_CPU_ARG && 117*0Sstevel@tonic-gate !(arg2val & (1 << cpup->cpu_id))) 118*0Sstevel@tonic-gate return (DDI_INTR_UNCLAIMED); 119*0Sstevel@tonic-gate ASSERT(arg2val == CAPTURE_CPU_ARG); 120*0Sstevel@tonic-gate if (cpup->cpu_m.xc_pend[pri] == 0) 121*0Sstevel@tonic-gate return (DDI_INTR_UNCLAIMED); 122*0Sstevel@tonic-gate 123*0Sstevel@tonic-gate cpup->cpu_m.xc_pend[X_CALL_MEDPRI] = 0; 124*0Sstevel@tonic-gate cpup->cpu_m.xc_ack[X_CALL_MEDPRI] = 1; 125*0Sstevel@tonic-gate 126*0Sstevel@tonic-gate for (;;) { 127*0Sstevel@tonic-gate if ((cpup->cpu_m.xc_state[X_CALL_MEDPRI] == XC_DONE) || 128*0Sstevel@tonic-gate (cpup->cpu_m.xc_pend[X_CALL_MEDPRI])) 129*0Sstevel@tonic-gate break; 130*0Sstevel@tonic-gate ht_pause(); 131*0Sstevel@tonic-gate return_instr(); 132*0Sstevel@tonic-gate } 133*0Sstevel@tonic-gate return (DDI_INTR_CLAIMED); 134*0Sstevel@tonic-gate } 135*0Sstevel@tonic-gate if (cpup->cpu_m.xc_pend[pri] == 0) 136*0Sstevel@tonic-gate return (DDI_INTR_UNCLAIMED); 137*0Sstevel@tonic-gate 138*0Sstevel@tonic-gate cpup->cpu_m.xc_pend[pri] = 0; 139*0Sstevel@tonic-gate op = cpup->cpu_m.xc_state[pri]; 140*0Sstevel@tonic-gate 141*0Sstevel@tonic-gate /* 142*0Sstevel@tonic-gate * When invalidating TLB entries, wait until the initiator changes the 143*0Sstevel@tonic-gate * memory PTE before doing any INVLPG. Otherwise, if the PTE in memory 144*0Sstevel@tonic-gate * hasn't been changed, the processor's TLB Flush filter may ignore 145*0Sstevel@tonic-gate * the INVLPG instruction. 146*0Sstevel@tonic-gate */ 147*0Sstevel@tonic-gate tlbflush = (cpup->cpu_m.xc_wait[pri] == 2); 148*0Sstevel@tonic-gate 149*0Sstevel@tonic-gate /* 150*0Sstevel@tonic-gate * Don't invoke a null function. 151*0Sstevel@tonic-gate */ 152*0Sstevel@tonic-gate if (xc_mboxes[pri].func != NULL) { 153*0Sstevel@tonic-gate if (!tlbflush) 154*0Sstevel@tonic-gate cpup->cpu_m.xc_retval[pri] = (*xc_mboxes[pri].func) 155*0Sstevel@tonic-gate (xc_mboxes[pri].arg1, xc_mboxes[pri].arg2, 156*0Sstevel@tonic-gate xc_mboxes[pri].arg3); 157*0Sstevel@tonic-gate } else 158*0Sstevel@tonic-gate cpup->cpu_m.xc_retval[pri] = 0; 159*0Sstevel@tonic-gate 160*0Sstevel@tonic-gate /* 161*0Sstevel@tonic-gate * Acknowledge that we have completed the x-call operation. 162*0Sstevel@tonic-gate */ 163*0Sstevel@tonic-gate cpup->cpu_m.xc_ack[pri] = 1; 164*0Sstevel@tonic-gate 165*0Sstevel@tonic-gate if (op == XC_CALL_OP) 166*0Sstevel@tonic-gate return (DDI_INTR_CLAIMED); 167*0Sstevel@tonic-gate 168*0Sstevel@tonic-gate /* 169*0Sstevel@tonic-gate * for (op == XC_SYNC_OP) 170*0Sstevel@tonic-gate * Wait for the initiator of the x-call to indicate 171*0Sstevel@tonic-gate * that all CPUs involved can proceed. 172*0Sstevel@tonic-gate */ 173*0Sstevel@tonic-gate while (cpup->cpu_m.xc_wait[pri]) { 174*0Sstevel@tonic-gate ht_pause(); 175*0Sstevel@tonic-gate return_instr(); 176*0Sstevel@tonic-gate } 177*0Sstevel@tonic-gate 178*0Sstevel@tonic-gate while (cpup->cpu_m.xc_state[pri] != XC_DONE) { 179*0Sstevel@tonic-gate ht_pause(); 180*0Sstevel@tonic-gate return_instr(); 181*0Sstevel@tonic-gate } 182*0Sstevel@tonic-gate 183*0Sstevel@tonic-gate /* 184*0Sstevel@tonic-gate * Flush the TLB, if that's what is requested. 185*0Sstevel@tonic-gate */ 186*0Sstevel@tonic-gate if (xc_mboxes[pri].func != NULL && tlbflush) { 187*0Sstevel@tonic-gate cpup->cpu_m.xc_retval[pri] = (*xc_mboxes[pri].func) 188*0Sstevel@tonic-gate (xc_mboxes[pri].arg1, xc_mboxes[pri].arg2, 189*0Sstevel@tonic-gate xc_mboxes[pri].arg3); 190*0Sstevel@tonic-gate } 191*0Sstevel@tonic-gate 192*0Sstevel@tonic-gate /* 193*0Sstevel@tonic-gate * Acknowledge that we have received the directive to continue. 194*0Sstevel@tonic-gate */ 195*0Sstevel@tonic-gate ASSERT(cpup->cpu_m.xc_ack[pri] == 0); 196*0Sstevel@tonic-gate cpup->cpu_m.xc_ack[pri] = 1; 197*0Sstevel@tonic-gate 198*0Sstevel@tonic-gate return (DDI_INTR_CLAIMED); 199*0Sstevel@tonic-gate } 200*0Sstevel@tonic-gate 201*0Sstevel@tonic-gate 202*0Sstevel@tonic-gate /* 203*0Sstevel@tonic-gate * xc_do_call: 204*0Sstevel@tonic-gate */ 205*0Sstevel@tonic-gate static void 206*0Sstevel@tonic-gate xc_do_call( 207*0Sstevel@tonic-gate xc_arg_t arg1, 208*0Sstevel@tonic-gate xc_arg_t arg2, 209*0Sstevel@tonic-gate xc_arg_t arg3, 210*0Sstevel@tonic-gate int pri, 211*0Sstevel@tonic-gate cpuset_t set, 212*0Sstevel@tonic-gate xc_func_t func, 213*0Sstevel@tonic-gate int sync) 214*0Sstevel@tonic-gate { 215*0Sstevel@tonic-gate /* 216*0Sstevel@tonic-gate * If the pri indicates a low priority lock (below LOCK_LEVEL), 217*0Sstevel@tonic-gate * we must disable preemption to avoid migrating to another CPU 218*0Sstevel@tonic-gate * during the call. 219*0Sstevel@tonic-gate */ 220*0Sstevel@tonic-gate if (pri == X_CALL_LOPRI) { 221*0Sstevel@tonic-gate kpreempt_disable(); 222*0Sstevel@tonic-gate } else { 223*0Sstevel@tonic-gate pri = X_CALL_HIPRI; 224*0Sstevel@tonic-gate } 225*0Sstevel@tonic-gate 226*0Sstevel@tonic-gate /* always grab highest mutex to avoid deadlock */ 227*0Sstevel@tonic-gate mutex_enter(&xc_mbox_lock[X_CALL_HIPRI]); 228*0Sstevel@tonic-gate xc_common(func, arg1, arg2, arg3, pri, set, sync); 229*0Sstevel@tonic-gate mutex_exit(&xc_mbox_lock[X_CALL_HIPRI]); 230*0Sstevel@tonic-gate if (pri == X_CALL_LOPRI) 231*0Sstevel@tonic-gate kpreempt_enable(); 232*0Sstevel@tonic-gate } 233*0Sstevel@tonic-gate 234*0Sstevel@tonic-gate 235*0Sstevel@tonic-gate /* 236*0Sstevel@tonic-gate * xc_call: call specified function on all processors 237*0Sstevel@tonic-gate * remotes may continue after service 238*0Sstevel@tonic-gate * we wait here until everybody has completed. 239*0Sstevel@tonic-gate */ 240*0Sstevel@tonic-gate void 241*0Sstevel@tonic-gate xc_call( 242*0Sstevel@tonic-gate xc_arg_t arg1, 243*0Sstevel@tonic-gate xc_arg_t arg2, 244*0Sstevel@tonic-gate xc_arg_t arg3, 245*0Sstevel@tonic-gate int pri, 246*0Sstevel@tonic-gate cpuset_t set, 247*0Sstevel@tonic-gate xc_func_t func) 248*0Sstevel@tonic-gate { 249*0Sstevel@tonic-gate xc_do_call(arg1, arg2, arg3, pri, set, func, 0); 250*0Sstevel@tonic-gate } 251*0Sstevel@tonic-gate 252*0Sstevel@tonic-gate /* 253*0Sstevel@tonic-gate * xc_sync: call specified function on all processors 254*0Sstevel@tonic-gate * after doing work, each remote waits until we let 255*0Sstevel@tonic-gate * it continue; send the contiunue after everyone has 256*0Sstevel@tonic-gate * informed us that they are done. 257*0Sstevel@tonic-gate */ 258*0Sstevel@tonic-gate void 259*0Sstevel@tonic-gate xc_sync( 260*0Sstevel@tonic-gate xc_arg_t arg1, 261*0Sstevel@tonic-gate xc_arg_t arg2, 262*0Sstevel@tonic-gate xc_arg_t arg3, 263*0Sstevel@tonic-gate int pri, 264*0Sstevel@tonic-gate cpuset_t set, 265*0Sstevel@tonic-gate xc_func_t func) 266*0Sstevel@tonic-gate { 267*0Sstevel@tonic-gate xc_do_call(arg1, arg2, arg3, pri, set, func, 1); 268*0Sstevel@tonic-gate } 269*0Sstevel@tonic-gate 270*0Sstevel@tonic-gate /* 271*0Sstevel@tonic-gate * xc_sync_wait: similar to xc_sync(), except that the starting 272*0Sstevel@tonic-gate * cpu waits for all other cpus to check in before running its 273*0Sstevel@tonic-gate * service locally. 274*0Sstevel@tonic-gate */ 275*0Sstevel@tonic-gate void 276*0Sstevel@tonic-gate xc_wait_sync( 277*0Sstevel@tonic-gate xc_arg_t arg1, 278*0Sstevel@tonic-gate xc_arg_t arg2, 279*0Sstevel@tonic-gate xc_arg_t arg3, 280*0Sstevel@tonic-gate int pri, 281*0Sstevel@tonic-gate cpuset_t set, 282*0Sstevel@tonic-gate xc_func_t func) 283*0Sstevel@tonic-gate { 284*0Sstevel@tonic-gate xc_do_call(arg1, arg2, arg3, pri, set, func, 2); 285*0Sstevel@tonic-gate } 286*0Sstevel@tonic-gate 287*0Sstevel@tonic-gate 288*0Sstevel@tonic-gate /* 289*0Sstevel@tonic-gate * The routines xc_capture_cpus and xc_release_cpus 290*0Sstevel@tonic-gate * can be used in place of xc_sync in order to implement a critical 291*0Sstevel@tonic-gate * code section where all CPUs in the system can be controlled. 292*0Sstevel@tonic-gate * xc_capture_cpus is used to start the critical code section, and 293*0Sstevel@tonic-gate * xc_release_cpus is used to end the critical code section. 294*0Sstevel@tonic-gate */ 295*0Sstevel@tonic-gate 296*0Sstevel@tonic-gate /* 297*0Sstevel@tonic-gate * Capture the CPUs specified in order to start a x-call session, 298*0Sstevel@tonic-gate * and/or to begin a critical section. 299*0Sstevel@tonic-gate */ 300*0Sstevel@tonic-gate void 301*0Sstevel@tonic-gate xc_capture_cpus(cpuset_t set) 302*0Sstevel@tonic-gate { 303*0Sstevel@tonic-gate int cix; 304*0Sstevel@tonic-gate int lcx; 305*0Sstevel@tonic-gate struct cpu *cpup; 306*0Sstevel@tonic-gate int i; 307*0Sstevel@tonic-gate cpuset_t *cpus; 308*0Sstevel@tonic-gate cpuset_t c; 309*0Sstevel@tonic-gate 310*0Sstevel@tonic-gate CPU_STATS_ADDQ(CPU, sys, xcalls, 1); 311*0Sstevel@tonic-gate 312*0Sstevel@tonic-gate /* 313*0Sstevel@tonic-gate * Prevent deadlocks where we take an interrupt and are waiting 314*0Sstevel@tonic-gate * for a mutex owned by one of the CPUs that is captured for 315*0Sstevel@tonic-gate * the x-call, while that CPU is waiting for some x-call signal 316*0Sstevel@tonic-gate * to be set by us. 317*0Sstevel@tonic-gate * 318*0Sstevel@tonic-gate * This mutex also prevents preemption, since it raises SPL above 319*0Sstevel@tonic-gate * LOCK_LEVEL (it is a spin-type driver mutex). 320*0Sstevel@tonic-gate */ 321*0Sstevel@tonic-gate /* always grab highest mutex to avoid deadlock */ 322*0Sstevel@tonic-gate mutex_enter(&xc_mbox_lock[X_CALL_HIPRI]); 323*0Sstevel@tonic-gate lcx = CPU->cpu_id; /* now we're safe */ 324*0Sstevel@tonic-gate 325*0Sstevel@tonic-gate ASSERT(CPU->cpu_flags & CPU_READY); 326*0Sstevel@tonic-gate 327*0Sstevel@tonic-gate /* 328*0Sstevel@tonic-gate * Wait for all cpus 329*0Sstevel@tonic-gate */ 330*0Sstevel@tonic-gate cpus = (cpuset_t *)&xc_mboxes[X_CALL_MEDPRI].arg2; 331*0Sstevel@tonic-gate if (CPU_IN_SET(*cpus, CPU->cpu_id)) 332*0Sstevel@tonic-gate CPUSET_ATOMIC_DEL(*cpus, CPU->cpu_id); 333*0Sstevel@tonic-gate for (;;) { 334*0Sstevel@tonic-gate c = *(volatile cpuset_t *)cpus; 335*0Sstevel@tonic-gate CPUSET_AND(c, cpu_ready_set); 336*0Sstevel@tonic-gate if (CPUSET_ISNULL(c)) 337*0Sstevel@tonic-gate break; 338*0Sstevel@tonic-gate ht_pause(); 339*0Sstevel@tonic-gate } 340*0Sstevel@tonic-gate 341*0Sstevel@tonic-gate /* 342*0Sstevel@tonic-gate * Store the set of CPUs involved in the x-call session, so that 343*0Sstevel@tonic-gate * xc_release_cpus will know what CPUs to act upon. 344*0Sstevel@tonic-gate */ 345*0Sstevel@tonic-gate xc_mboxes[X_CALL_MEDPRI].set = set; 346*0Sstevel@tonic-gate xc_mboxes[X_CALL_MEDPRI].arg2 = CAPTURE_CPU_ARG; 347*0Sstevel@tonic-gate 348*0Sstevel@tonic-gate /* 349*0Sstevel@tonic-gate * Now capture each CPU in the set and cause it to go into a 350*0Sstevel@tonic-gate * holding pattern. 351*0Sstevel@tonic-gate */ 352*0Sstevel@tonic-gate i = 0; 353*0Sstevel@tonic-gate for (cix = 0; cix < NCPU; cix++) { 354*0Sstevel@tonic-gate if ((cpup = cpu[cix]) == NULL || 355*0Sstevel@tonic-gate (cpup->cpu_flags & CPU_READY) == 0) { 356*0Sstevel@tonic-gate /* 357*0Sstevel@tonic-gate * In case CPU wasn't ready, but becomes ready later, 358*0Sstevel@tonic-gate * take the CPU out of the set now. 359*0Sstevel@tonic-gate */ 360*0Sstevel@tonic-gate CPUSET_DEL(set, cix); 361*0Sstevel@tonic-gate continue; 362*0Sstevel@tonic-gate } 363*0Sstevel@tonic-gate if (cix != lcx && CPU_IN_SET(set, cix)) { 364*0Sstevel@tonic-gate cpup->cpu_m.xc_ack[X_CALL_MEDPRI] = 0; 365*0Sstevel@tonic-gate cpup->cpu_m.xc_state[X_CALL_MEDPRI] = XC_HOLD; 366*0Sstevel@tonic-gate cpup->cpu_m.xc_pend[X_CALL_MEDPRI] = 1; 367*0Sstevel@tonic-gate send_dirint(cix, XC_MED_PIL); 368*0Sstevel@tonic-gate } 369*0Sstevel@tonic-gate i++; 370*0Sstevel@tonic-gate if (i >= ncpus) 371*0Sstevel@tonic-gate break; 372*0Sstevel@tonic-gate } 373*0Sstevel@tonic-gate 374*0Sstevel@tonic-gate /* 375*0Sstevel@tonic-gate * Wait here until all remote calls to complete. 376*0Sstevel@tonic-gate */ 377*0Sstevel@tonic-gate i = 0; 378*0Sstevel@tonic-gate for (cix = 0; cix < NCPU; cix++) { 379*0Sstevel@tonic-gate if (lcx != cix && CPU_IN_SET(set, cix)) { 380*0Sstevel@tonic-gate cpup = cpu[cix]; 381*0Sstevel@tonic-gate while (cpup->cpu_m.xc_ack[X_CALL_MEDPRI] == 0) { 382*0Sstevel@tonic-gate ht_pause(); 383*0Sstevel@tonic-gate return_instr(); 384*0Sstevel@tonic-gate } 385*0Sstevel@tonic-gate cpup->cpu_m.xc_ack[X_CALL_MEDPRI] = 0; 386*0Sstevel@tonic-gate } 387*0Sstevel@tonic-gate i++; 388*0Sstevel@tonic-gate if (i >= ncpus) 389*0Sstevel@tonic-gate break; 390*0Sstevel@tonic-gate } 391*0Sstevel@tonic-gate 392*0Sstevel@tonic-gate } 393*0Sstevel@tonic-gate 394*0Sstevel@tonic-gate /* 395*0Sstevel@tonic-gate * Release the CPUs captured by xc_capture_cpus, thus terminating the 396*0Sstevel@tonic-gate * x-call session and exiting the critical section. 397*0Sstevel@tonic-gate */ 398*0Sstevel@tonic-gate void 399*0Sstevel@tonic-gate xc_release_cpus(void) 400*0Sstevel@tonic-gate { 401*0Sstevel@tonic-gate int cix; 402*0Sstevel@tonic-gate int lcx = (int)(CPU->cpu_id); 403*0Sstevel@tonic-gate cpuset_t set = xc_mboxes[X_CALL_MEDPRI].set; 404*0Sstevel@tonic-gate struct cpu *cpup; 405*0Sstevel@tonic-gate int i; 406*0Sstevel@tonic-gate 407*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&xc_mbox_lock[X_CALL_HIPRI])); 408*0Sstevel@tonic-gate 409*0Sstevel@tonic-gate /* 410*0Sstevel@tonic-gate * Allow each CPU to exit its holding pattern. 411*0Sstevel@tonic-gate */ 412*0Sstevel@tonic-gate i = 0; 413*0Sstevel@tonic-gate for (cix = 0; cix < NCPU; cix++) { 414*0Sstevel@tonic-gate if ((cpup = cpu[cix]) == NULL) 415*0Sstevel@tonic-gate continue; 416*0Sstevel@tonic-gate if ((cpup->cpu_flags & CPU_READY) && 417*0Sstevel@tonic-gate (cix != lcx) && CPU_IN_SET(set, cix)) { 418*0Sstevel@tonic-gate /* 419*0Sstevel@tonic-gate * Clear xc_ack since we will be waiting for it 420*0Sstevel@tonic-gate * to be set again after we set XC_DONE. 421*0Sstevel@tonic-gate */ 422*0Sstevel@tonic-gate cpup->cpu_m.xc_state[X_CALL_MEDPRI] = XC_DONE; 423*0Sstevel@tonic-gate } 424*0Sstevel@tonic-gate i++; 425*0Sstevel@tonic-gate if (i >= ncpus) 426*0Sstevel@tonic-gate break; 427*0Sstevel@tonic-gate } 428*0Sstevel@tonic-gate 429*0Sstevel@tonic-gate xc_mboxes[X_CALL_MEDPRI].arg2 = 0; 430*0Sstevel@tonic-gate mutex_exit(&xc_mbox_lock[X_CALL_HIPRI]); 431*0Sstevel@tonic-gate } 432*0Sstevel@tonic-gate 433*0Sstevel@tonic-gate /* 434*0Sstevel@tonic-gate * Common code to call a specified function on a set of processors. 435*0Sstevel@tonic-gate * sync specifies what kind of waiting is done. 436*0Sstevel@tonic-gate * -1 - no waiting, don't release remotes 437*0Sstevel@tonic-gate * 0 - no waiting, release remotes immediately 438*0Sstevel@tonic-gate * 1 - run service locally w/o waiting for remotes. 439*0Sstevel@tonic-gate * 2 - wait for remotes before running locally 440*0Sstevel@tonic-gate */ 441*0Sstevel@tonic-gate static void 442*0Sstevel@tonic-gate xc_common( 443*0Sstevel@tonic-gate xc_func_t func, 444*0Sstevel@tonic-gate xc_arg_t arg1, 445*0Sstevel@tonic-gate xc_arg_t arg2, 446*0Sstevel@tonic-gate xc_arg_t arg3, 447*0Sstevel@tonic-gate int pri, 448*0Sstevel@tonic-gate cpuset_t set, 449*0Sstevel@tonic-gate int sync) 450*0Sstevel@tonic-gate { 451*0Sstevel@tonic-gate int cix; 452*0Sstevel@tonic-gate int lcx = (int)(CPU->cpu_id); 453*0Sstevel@tonic-gate struct cpu *cpup; 454*0Sstevel@tonic-gate 455*0Sstevel@tonic-gate ASSERT(panicstr == NULL); 456*0Sstevel@tonic-gate 457*0Sstevel@tonic-gate ASSERT(MUTEX_HELD(&xc_mbox_lock[X_CALL_HIPRI])); 458*0Sstevel@tonic-gate ASSERT(CPU->cpu_flags & CPU_READY); 459*0Sstevel@tonic-gate 460*0Sstevel@tonic-gate /* 461*0Sstevel@tonic-gate * Set up the service definition mailbox. 462*0Sstevel@tonic-gate */ 463*0Sstevel@tonic-gate xc_mboxes[pri].func = func; 464*0Sstevel@tonic-gate xc_mboxes[pri].arg1 = arg1; 465*0Sstevel@tonic-gate xc_mboxes[pri].arg2 = arg2; 466*0Sstevel@tonic-gate xc_mboxes[pri].arg3 = arg3; 467*0Sstevel@tonic-gate 468*0Sstevel@tonic-gate /* 469*0Sstevel@tonic-gate * Request service on all remote processors. 470*0Sstevel@tonic-gate */ 471*0Sstevel@tonic-gate for (cix = 0; cix < NCPU; cix++) { 472*0Sstevel@tonic-gate if ((cpup = cpu[cix]) == NULL || 473*0Sstevel@tonic-gate (cpup->cpu_flags & CPU_READY) == 0) { 474*0Sstevel@tonic-gate /* 475*0Sstevel@tonic-gate * In case CPU wasn't ready, but becomes ready later, 476*0Sstevel@tonic-gate * take the CPU out of the set now. 477*0Sstevel@tonic-gate */ 478*0Sstevel@tonic-gate CPUSET_DEL(set, cix); 479*0Sstevel@tonic-gate } else if (cix != lcx && CPU_IN_SET(set, cix)) { 480*0Sstevel@tonic-gate CPU_STATS_ADDQ(CPU, sys, xcalls, 1); 481*0Sstevel@tonic-gate cpup->cpu_m.xc_ack[pri] = 0; 482*0Sstevel@tonic-gate cpup->cpu_m.xc_wait[pri] = sync; 483*0Sstevel@tonic-gate if (sync > 0) 484*0Sstevel@tonic-gate cpup->cpu_m.xc_state[pri] = XC_SYNC_OP; 485*0Sstevel@tonic-gate else 486*0Sstevel@tonic-gate cpup->cpu_m.xc_state[pri] = XC_CALL_OP; 487*0Sstevel@tonic-gate cpup->cpu_m.xc_pend[pri] = 1; 488*0Sstevel@tonic-gate send_dirint(cix, xc_xlat_xcptoipl[pri]); 489*0Sstevel@tonic-gate } 490*0Sstevel@tonic-gate } 491*0Sstevel@tonic-gate 492*0Sstevel@tonic-gate /* 493*0Sstevel@tonic-gate * Run service locally if not waiting for remotes. 494*0Sstevel@tonic-gate */ 495*0Sstevel@tonic-gate if (sync != 2 && CPU_IN_SET(set, lcx) && func != NULL) 496*0Sstevel@tonic-gate CPU->cpu_m.xc_retval[pri] = (*func)(arg1, arg2, arg3); 497*0Sstevel@tonic-gate 498*0Sstevel@tonic-gate if (sync == -1) 499*0Sstevel@tonic-gate return; 500*0Sstevel@tonic-gate 501*0Sstevel@tonic-gate /* 502*0Sstevel@tonic-gate * Wait here until all remote calls complete. 503*0Sstevel@tonic-gate */ 504*0Sstevel@tonic-gate for (cix = 0; cix < NCPU; cix++) { 505*0Sstevel@tonic-gate if (lcx != cix && CPU_IN_SET(set, cix)) { 506*0Sstevel@tonic-gate cpup = cpu[cix]; 507*0Sstevel@tonic-gate while (cpup->cpu_m.xc_ack[pri] == 0) { 508*0Sstevel@tonic-gate ht_pause(); 509*0Sstevel@tonic-gate return_instr(); 510*0Sstevel@tonic-gate } 511*0Sstevel@tonic-gate cpup->cpu_m.xc_ack[pri] = 0; 512*0Sstevel@tonic-gate } 513*0Sstevel@tonic-gate } 514*0Sstevel@tonic-gate 515*0Sstevel@tonic-gate /* 516*0Sstevel@tonic-gate * Run service locally if waiting for remotes. 517*0Sstevel@tonic-gate */ 518*0Sstevel@tonic-gate if (sync == 2 && CPU_IN_SET(set, lcx) && func != NULL) 519*0Sstevel@tonic-gate CPU->cpu_m.xc_retval[pri] = (*func)(arg1, arg2, arg3); 520*0Sstevel@tonic-gate 521*0Sstevel@tonic-gate if (sync == 0) 522*0Sstevel@tonic-gate return; 523*0Sstevel@tonic-gate 524*0Sstevel@tonic-gate /* 525*0Sstevel@tonic-gate * Release any waiting CPUs 526*0Sstevel@tonic-gate */ 527*0Sstevel@tonic-gate for (cix = 0; cix < NCPU; cix++) { 528*0Sstevel@tonic-gate if (lcx != cix && CPU_IN_SET(set, cix)) { 529*0Sstevel@tonic-gate cpup = cpu[cix]; 530*0Sstevel@tonic-gate if (cpup != NULL && (cpup->cpu_flags & CPU_READY)) { 531*0Sstevel@tonic-gate cpup->cpu_m.xc_wait[pri] = 0; 532*0Sstevel@tonic-gate cpup->cpu_m.xc_state[pri] = XC_DONE; 533*0Sstevel@tonic-gate } 534*0Sstevel@tonic-gate } 535*0Sstevel@tonic-gate } 536*0Sstevel@tonic-gate 537*0Sstevel@tonic-gate /* 538*0Sstevel@tonic-gate * Wait for all CPUs to acknowledge completion before we continue. 539*0Sstevel@tonic-gate * Without this check it's possible (on a VM or hyper-threaded CPUs 540*0Sstevel@tonic-gate * or in the presence of Service Management Interrupts which can all 541*0Sstevel@tonic-gate * cause delays) for the remote processor to still be waiting by 542*0Sstevel@tonic-gate * the time xc_common() is next invoked with the sync flag set 543*0Sstevel@tonic-gate * resulting in a deadlock. 544*0Sstevel@tonic-gate */ 545*0Sstevel@tonic-gate for (cix = 0; cix < NCPU; cix++) { 546*0Sstevel@tonic-gate if (lcx != cix && CPU_IN_SET(set, cix)) { 547*0Sstevel@tonic-gate cpup = cpu[cix]; 548*0Sstevel@tonic-gate if (cpup != NULL && (cpup->cpu_flags & CPU_READY)) { 549*0Sstevel@tonic-gate while (cpup->cpu_m.xc_ack[pri] == 0) { 550*0Sstevel@tonic-gate ht_pause(); 551*0Sstevel@tonic-gate return_instr(); 552*0Sstevel@tonic-gate } 553*0Sstevel@tonic-gate cpup->cpu_m.xc_ack[pri] = 0; 554*0Sstevel@tonic-gate } 555*0Sstevel@tonic-gate } 556*0Sstevel@tonic-gate } 557*0Sstevel@tonic-gate } 558*0Sstevel@tonic-gate 559*0Sstevel@tonic-gate /* 560*0Sstevel@tonic-gate * xc_trycall: attempt to call specified function on all processors 561*0Sstevel@tonic-gate * remotes may wait for a long time 562*0Sstevel@tonic-gate * we continue immediately 563*0Sstevel@tonic-gate */ 564*0Sstevel@tonic-gate void 565*0Sstevel@tonic-gate xc_trycall( 566*0Sstevel@tonic-gate xc_arg_t arg1, 567*0Sstevel@tonic-gate xc_arg_t arg2, 568*0Sstevel@tonic-gate xc_arg_t arg3, 569*0Sstevel@tonic-gate cpuset_t set, 570*0Sstevel@tonic-gate xc_func_t func) 571*0Sstevel@tonic-gate { 572*0Sstevel@tonic-gate int save_kernel_preemption; 573*0Sstevel@tonic-gate extern int IGNORE_KERNEL_PREEMPTION; 574*0Sstevel@tonic-gate 575*0Sstevel@tonic-gate /* 576*0Sstevel@tonic-gate * If we can grab the mutex, we'll do the cross-call. If not -- if 577*0Sstevel@tonic-gate * someone else is already doing a cross-call -- we won't. 578*0Sstevel@tonic-gate */ 579*0Sstevel@tonic-gate 580*0Sstevel@tonic-gate save_kernel_preemption = IGNORE_KERNEL_PREEMPTION; 581*0Sstevel@tonic-gate IGNORE_KERNEL_PREEMPTION = 1; 582*0Sstevel@tonic-gate if (mutex_tryenter(&xc_mbox_lock[X_CALL_HIPRI])) { 583*0Sstevel@tonic-gate xc_common(func, arg1, arg2, arg3, X_CALL_HIPRI, set, -1); 584*0Sstevel@tonic-gate mutex_exit(&xc_mbox_lock[X_CALL_HIPRI]); 585*0Sstevel@tonic-gate } 586*0Sstevel@tonic-gate IGNORE_KERNEL_PREEMPTION = save_kernel_preemption; 587*0Sstevel@tonic-gate } 588*0Sstevel@tonic-gate 589*0Sstevel@tonic-gate /* 590*0Sstevel@tonic-gate * Used by the debugger to cross-call the other CPUs, thus causing them to 591*0Sstevel@tonic-gate * enter the debugger. We can't hold locks, so we spin on the cross-call 592*0Sstevel@tonic-gate * lock until we get it. When we get it, we send the cross-call, and assume 593*0Sstevel@tonic-gate * that we successfully stopped the other CPUs. 594*0Sstevel@tonic-gate */ 595*0Sstevel@tonic-gate void 596*0Sstevel@tonic-gate kdi_xc_others(int this_cpu, void (*func)(void)) 597*0Sstevel@tonic-gate { 598*0Sstevel@tonic-gate extern int IGNORE_KERNEL_PREEMPTION; 599*0Sstevel@tonic-gate int save_kernel_preemption; 600*0Sstevel@tonic-gate mutex_impl_t *lp; 601*0Sstevel@tonic-gate cpuset_t set; 602*0Sstevel@tonic-gate int x; 603*0Sstevel@tonic-gate 604*0Sstevel@tonic-gate CPUSET_ALL_BUT(set, this_cpu); 605*0Sstevel@tonic-gate 606*0Sstevel@tonic-gate save_kernel_preemption = IGNORE_KERNEL_PREEMPTION; 607*0Sstevel@tonic-gate IGNORE_KERNEL_PREEMPTION = 1; 608*0Sstevel@tonic-gate 609*0Sstevel@tonic-gate lp = (mutex_impl_t *)&xc_mbox_lock[X_CALL_HIPRI]; 610*0Sstevel@tonic-gate for (x = 0; x < 0x400000; x++) { 611*0Sstevel@tonic-gate if (lock_spin_try(&lp->m_spin.m_spinlock)) { 612*0Sstevel@tonic-gate xc_common((xc_func_t)func, 0, 0, 0, X_CALL_HIPRI, 613*0Sstevel@tonic-gate set, -1); 614*0Sstevel@tonic-gate lp->m_spin.m_spinlock = 0; /* XXX */ 615*0Sstevel@tonic-gate break; 616*0Sstevel@tonic-gate } 617*0Sstevel@tonic-gate (void) xc_serv((caddr_t)X_CALL_MEDPRI, NULL); 618*0Sstevel@tonic-gate } 619*0Sstevel@tonic-gate IGNORE_KERNEL_PREEMPTION = save_kernel_preemption; 620*0Sstevel@tonic-gate } 621