11642Sgavinm /* 21642Sgavinm * CDDL HEADER START 31642Sgavinm * 41642Sgavinm * The contents of this file are subject to the terms of the 51642Sgavinm * Common Development and Distribution License (the "License"). 61642Sgavinm * You may not use this file except in compliance with the License. 71642Sgavinm * 81642Sgavinm * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 91642Sgavinm * or http://www.opensolaris.org/os/licensing. 101642Sgavinm * See the License for the specific language governing permissions 111642Sgavinm * and limitations under the License. 121642Sgavinm * 131642Sgavinm * When distributing Covered Code, include this CDDL HEADER in each 141642Sgavinm * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 151642Sgavinm * If applicable, add the following below this CDDL HEADER, with the 161642Sgavinm * fields enclosed by brackets "[]" replaced with your own identifying 171642Sgavinm * information: Portions Copyright [yyyy] [name of copyright owner] 181642Sgavinm * 191642Sgavinm * CDDL HEADER END 201642Sgavinm */ 211642Sgavinm 221414Scindi /* 237349SAdrian.Frost@Sun.COM * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 241414Scindi * Use is subject to license terms. 251414Scindi */ 261414Scindi 271414Scindi /* 281414Scindi * Public interface to routines implemented by CPU modules 291414Scindi */ 301414Scindi 315254Sgavinm #include <sys/types.h> 325254Sgavinm #include <sys/atomic.h> 331414Scindi #include <sys/x86_archext.h> 341414Scindi #include <sys/cpu_module_impl.h> 355254Sgavinm #include <sys/cpu_module_ms.h> 361414Scindi #include <sys/fm/util.h> 371414Scindi #include <sys/reboot.h> 381414Scindi #include <sys/modctl.h> 391414Scindi #include <sys/param.h> 401414Scindi #include <sys/cmn_err.h> 411414Scindi #include <sys/systm.h> 425254Sgavinm #include <sys/fm/protocol.h> 435254Sgavinm #include <sys/pcb.h> 445254Sgavinm #include <sys/ontrap.h> 455254Sgavinm #include <sys/psw.h> 465254Sgavinm #include <sys/privregs.h> 47*7532SSean.Ye@Sun.COM #include <sys/machsystm.h> 481414Scindi 495254Sgavinm /* 505254Sgavinm * Set to force cmi_init to fail. 515254Sgavinm */ 525254Sgavinm int cmi_no_init = 0; 535254Sgavinm 545254Sgavinm /* 555254Sgavinm * Set to avoid MCA initialization. 565254Sgavinm */ 575254Sgavinm int cmi_no_mca_init = 0; 581414Scindi 591414Scindi /* 602869Sgavinm * If cleared for debugging we will not attempt to load a model-specific 612869Sgavinm * cpu module but will load the generic cpu module instead. 622869Sgavinm */ 632869Sgavinm int cmi_force_generic = 0; 642869Sgavinm 652869Sgavinm /* 661414Scindi * If cleared for debugging, we will suppress panicking on fatal hardware 671414Scindi * errors. This should *only* be used for debugging; it use can and will 681414Scindi * cause data corruption if actual hardware errors are detected by the system. 691414Scindi */ 701414Scindi int cmi_panic_on_uncorrectable_error = 1; 711414Scindi 72*7532SSean.Ye@Sun.COM #ifndef __xpv 735254Sgavinm /* 747349SAdrian.Frost@Sun.COM * Set to indicate whether we are able to enable cmci interrupt. 757349SAdrian.Frost@Sun.COM */ 767349SAdrian.Frost@Sun.COM int cmi_enable_cmci = 0; 77*7532SSean.Ye@Sun.COM #endif 787349SAdrian.Frost@Sun.COM 797349SAdrian.Frost@Sun.COM /* 805254Sgavinm * Subdirectory (relative to the module search path) in which we will 815254Sgavinm * look for cpu modules. 825254Sgavinm */ 835254Sgavinm #define CPUMOD_SUBDIR "cpu" 845254Sgavinm 855254Sgavinm /* 865254Sgavinm * CPU modules have a filenames such as "cpu.AuthenticAMD.15" and 875254Sgavinm * "cpu.generic" - the "cpu" prefix is specified by the following. 885254Sgavinm */ 895254Sgavinm #define CPUMOD_PREFIX "cpu" 905254Sgavinm 915254Sgavinm /* 925254Sgavinm * Structure used to keep track of cpu modules we have loaded and their ops 935254Sgavinm */ 945254Sgavinm typedef struct cmi { 955254Sgavinm struct cmi *cmi_next; 965254Sgavinm struct cmi *cmi_prev; 975254Sgavinm const cmi_ops_t *cmi_ops; 985254Sgavinm struct modctl *cmi_modp; 995254Sgavinm uint_t cmi_refcnt; 1005254Sgavinm } cmi_t; 1015254Sgavinm 1021414Scindi static cmi_t *cmi_list; 1031414Scindi static kmutex_t cmi_load_lock; 1041414Scindi 1055254Sgavinm /* 1065254Sgavinm * Functions we need from cmi_hw.c that are not part of the cpu_module.h 1075254Sgavinm * interface. 1085254Sgavinm */ 109*7532SSean.Ye@Sun.COM extern cmi_hdl_t cmi_hdl_create(enum cmi_hdl_class, uint_t, uint_t, uint_t); 1105254Sgavinm extern void cmi_hdl_setcmi(cmi_hdl_t, void *, void *); 1115254Sgavinm extern void *cmi_hdl_getcmi(cmi_hdl_t); 1125254Sgavinm extern void cmi_hdl_setmc(cmi_hdl_t, const struct cmi_mc_ops *, void *); 113*7532SSean.Ye@Sun.COM extern void cmi_hdl_inj_begin(cmi_hdl_t); 114*7532SSean.Ye@Sun.COM extern void cmi_hdl_inj_end(cmi_hdl_t); 1155254Sgavinm 1165254Sgavinm #define HDL2CMI(hdl) cmi_hdl_getcmi(hdl) 1175254Sgavinm 1185254Sgavinm #define CMI_OPS(cmi) (cmi)->cmi_ops 1195254Sgavinm #define CMI_OP_PRESENT(cmi, op) ((cmi) && CMI_OPS(cmi)->op != NULL) 1205254Sgavinm 1215254Sgavinm #define CMI_MATCH_VENDOR 0 /* Just match on vendor */ 1225254Sgavinm #define CMI_MATCH_FAMILY 1 /* Match down to family */ 1235254Sgavinm #define CMI_MATCH_MODEL 2 /* Match down to model */ 1245254Sgavinm #define CMI_MATCH_STEPPING 3 /* Match down to stepping */ 1255254Sgavinm 1265254Sgavinm static void 1275254Sgavinm cmi_link(cmi_t *cmi) 1285254Sgavinm { 1295254Sgavinm ASSERT(MUTEX_HELD(&cmi_load_lock)); 1305254Sgavinm 1315254Sgavinm cmi->cmi_prev = NULL; 1325254Sgavinm cmi->cmi_next = cmi_list; 1335254Sgavinm if (cmi_list != NULL) 1345254Sgavinm cmi_list->cmi_prev = cmi; 1355254Sgavinm cmi_list = cmi; 1365254Sgavinm } 1375254Sgavinm 1385254Sgavinm static void 1395254Sgavinm cmi_unlink(cmi_t *cmi) 1401414Scindi { 1415254Sgavinm ASSERT(MUTEX_HELD(&cmi_load_lock)); 1425254Sgavinm ASSERT(cmi->cmi_refcnt == 0); 1435254Sgavinm 1445254Sgavinm if (cmi->cmi_prev != NULL) 1455254Sgavinm cmi->cmi_prev = cmi->cmi_next; 1465254Sgavinm 1475254Sgavinm if (cmi->cmi_next != NULL) 1485254Sgavinm cmi->cmi_next->cmi_prev = cmi->cmi_prev; 1495254Sgavinm 1505254Sgavinm if (cmi_list == cmi) 1515254Sgavinm cmi_list = cmi->cmi_next; 1525254Sgavinm } 1535254Sgavinm 1545254Sgavinm /* 1555254Sgavinm * Hold the module in memory. We call to CPU modules without using the 1565254Sgavinm * stubs mechanism, so these modules must be manually held in memory. 1575254Sgavinm * The mod_ref acts as if another loaded module has a dependency on us. 1585254Sgavinm */ 1595254Sgavinm static void 1605254Sgavinm cmi_hold(cmi_t *cmi) 1615254Sgavinm { 1625254Sgavinm ASSERT(MUTEX_HELD(&cmi_load_lock)); 1635254Sgavinm 1645254Sgavinm mutex_enter(&mod_lock); 1655254Sgavinm cmi->cmi_modp->mod_ref++; 1665254Sgavinm mutex_exit(&mod_lock); 1675254Sgavinm cmi->cmi_refcnt++; 1685254Sgavinm } 1695254Sgavinm 1705254Sgavinm static void 1715254Sgavinm cmi_rele(cmi_t *cmi) 1725254Sgavinm { 1735254Sgavinm ASSERT(MUTEX_HELD(&cmi_load_lock)); 1745254Sgavinm 1755254Sgavinm mutex_enter(&mod_lock); 1765254Sgavinm cmi->cmi_modp->mod_ref--; 1775254Sgavinm mutex_exit(&mod_lock); 1785254Sgavinm 1795254Sgavinm if (--cmi->cmi_refcnt == 0) { 1805254Sgavinm cmi_unlink(cmi); 1815254Sgavinm kmem_free(cmi, sizeof (cmi_t)); 1825254Sgavinm } 1835254Sgavinm } 1845254Sgavinm 1855254Sgavinm static cmi_ops_t * 1865254Sgavinm cmi_getops(modctl_t *modp) 1875254Sgavinm { 1885254Sgavinm cmi_ops_t *ops; 1895254Sgavinm 1905254Sgavinm if ((ops = (cmi_ops_t *)modlookup_by_modctl(modp, "_cmi_ops")) == 1915254Sgavinm NULL) { 1925254Sgavinm cmn_err(CE_WARN, "cpu module '%s' is invalid: no _cmi_ops " 1935254Sgavinm "found", modp->mod_modname); 1945254Sgavinm return (NULL); 1955254Sgavinm } 1965254Sgavinm 1975254Sgavinm if (ops->cmi_init == NULL) { 1985254Sgavinm cmn_err(CE_WARN, "cpu module '%s' is invalid: no cmi_init " 1995254Sgavinm "entry point", modp->mod_modname); 2005254Sgavinm return (NULL); 2015254Sgavinm } 2025254Sgavinm 2035254Sgavinm return (ops); 2041414Scindi } 2051414Scindi 2061414Scindi static cmi_t * 2071414Scindi cmi_load_modctl(modctl_t *modp) 2081414Scindi { 2095254Sgavinm cmi_ops_t *ops; 2105254Sgavinm uintptr_t ver; 2111414Scindi cmi_t *cmi; 2125254Sgavinm cmi_api_ver_t apiver; 2131414Scindi 2141414Scindi ASSERT(MUTEX_HELD(&cmi_load_lock)); 2151414Scindi 2161414Scindi for (cmi = cmi_list; cmi != NULL; cmi = cmi->cmi_next) { 2171414Scindi if (cmi->cmi_modp == modp) 2181414Scindi return (cmi); 2191414Scindi } 2201414Scindi 2215254Sgavinm if ((ver = modlookup_by_modctl(modp, "_cmi_api_version")) == NULL) { 2225254Sgavinm /* 2235254Sgavinm * Apparently a cpu module before versioning was introduced - 2245254Sgavinm * we call this version 0. 2255254Sgavinm */ 2265254Sgavinm apiver = CMI_API_VERSION_0; 2275254Sgavinm } else { 2285254Sgavinm apiver = *((cmi_api_ver_t *)ver); 2295254Sgavinm if (!CMI_API_VERSION_CHKMAGIC(apiver)) { 2305254Sgavinm cmn_err(CE_WARN, "cpu module '%s' is invalid: " 2315254Sgavinm "_cmi_api_version 0x%x has bad magic", 2325254Sgavinm modp->mod_modname, apiver); 2335254Sgavinm return (NULL); 2345254Sgavinm } 2355254Sgavinm } 2365254Sgavinm 2375254Sgavinm if (apiver != CMI_API_VERSION) { 2385254Sgavinm cmn_err(CE_WARN, "cpu module '%s' has API version %d, " 2395254Sgavinm "kernel requires API version %d", modp->mod_modname, 2405254Sgavinm CMI_API_VERSION_TOPRINT(apiver), 2415254Sgavinm CMI_API_VERSION_TOPRINT(CMI_API_VERSION)); 2421414Scindi return (NULL); 2431414Scindi } 2441414Scindi 2455254Sgavinm if ((ops = cmi_getops(modp)) == NULL) 2465254Sgavinm return (NULL); 2471414Scindi 2485254Sgavinm cmi = kmem_zalloc(sizeof (*cmi), KM_SLEEP); 2495254Sgavinm cmi->cmi_ops = ops; 2501414Scindi cmi->cmi_modp = modp; 2511414Scindi 2525254Sgavinm cmi_link(cmi); 2535254Sgavinm 2545254Sgavinm return (cmi); 2555254Sgavinm } 2565254Sgavinm 2575254Sgavinm static int 2585254Sgavinm cmi_cpu_match(cmi_hdl_t hdl1, cmi_hdl_t hdl2, int match) 2595254Sgavinm { 2605254Sgavinm if (match >= CMI_MATCH_VENDOR && 2615254Sgavinm cmi_hdl_vendor(hdl1) != cmi_hdl_vendor(hdl2)) 2625254Sgavinm return (0); 2635254Sgavinm 2645254Sgavinm if (match >= CMI_MATCH_FAMILY && 2655254Sgavinm cmi_hdl_family(hdl1) != cmi_hdl_family(hdl2)) 2665254Sgavinm return (0); 2675254Sgavinm 2685254Sgavinm if (match >= CMI_MATCH_MODEL && 2695254Sgavinm cmi_hdl_model(hdl1) != cmi_hdl_model(hdl2)) 2705254Sgavinm return (0); 2715254Sgavinm 2725254Sgavinm if (match >= CMI_MATCH_STEPPING && 2735254Sgavinm cmi_hdl_stepping(hdl1) != cmi_hdl_stepping(hdl2)) 2745254Sgavinm return (0); 2755254Sgavinm 2765254Sgavinm return (1); 2775254Sgavinm } 2785254Sgavinm 2795254Sgavinm static int 2805254Sgavinm cmi_search_list_cb(cmi_hdl_t whdl, void *arg1, void *arg2, void *arg3) 2815254Sgavinm { 2825254Sgavinm cmi_hdl_t thdl = (cmi_hdl_t)arg1; 2835254Sgavinm int match = *((int *)arg2); 2845254Sgavinm cmi_hdl_t *rsltp = (cmi_hdl_t *)arg3; 2855254Sgavinm 2865254Sgavinm if (cmi_cpu_match(thdl, whdl, match)) { 2875254Sgavinm cmi_hdl_hold(whdl); /* short-term hold */ 2885254Sgavinm *rsltp = whdl; 2895254Sgavinm return (CMI_HDL_WALK_DONE); 2905254Sgavinm } else { 2915254Sgavinm return (CMI_HDL_WALK_NEXT); 2925254Sgavinm } 2935254Sgavinm } 2945254Sgavinm 2955254Sgavinm static cmi_t * 2965254Sgavinm cmi_search_list(cmi_hdl_t hdl, int match) 2975254Sgavinm { 2985254Sgavinm cmi_hdl_t dhdl = NULL; 2995254Sgavinm cmi_t *cmi = NULL; 3005254Sgavinm 3015254Sgavinm ASSERT(MUTEX_HELD(&cmi_load_lock)); 3025254Sgavinm 3035254Sgavinm cmi_hdl_walk(cmi_search_list_cb, (void *)hdl, (void *)&match, &dhdl); 3045254Sgavinm if (dhdl) { 3055254Sgavinm cmi = HDL2CMI(dhdl); 3065254Sgavinm cmi_hdl_rele(dhdl); /* held in cmi_search_list_cb */ 3075254Sgavinm } 3081414Scindi 3091414Scindi return (cmi); 3101414Scindi } 3111414Scindi 3121414Scindi static cmi_t * 3135254Sgavinm cmi_load_module(cmi_hdl_t hdl, int match, int *chosenp) 3141414Scindi { 3151414Scindi modctl_t *modp; 3161414Scindi cmi_t *cmi; 3175254Sgavinm int modid; 3181414Scindi uint_t s[3]; 3191414Scindi 3205254Sgavinm ASSERT(MUTEX_HELD(&cmi_load_lock)); 3215254Sgavinm ASSERT(match == CMI_MATCH_STEPPING || match == CMI_MATCH_MODEL || 3225254Sgavinm match == CMI_MATCH_FAMILY || match == CMI_MATCH_VENDOR); 3235254Sgavinm 3241414Scindi /* 3255254Sgavinm * Have we already loaded a module for a cpu with the same 3265254Sgavinm * vendor/family/model/stepping? 3271414Scindi */ 3285254Sgavinm if ((cmi = cmi_search_list(hdl, match)) != NULL) { 3295254Sgavinm cmi_hold(cmi); 3305254Sgavinm return (cmi); 3311414Scindi } 3321414Scindi 3335254Sgavinm s[0] = cmi_hdl_family(hdl); 3345254Sgavinm s[1] = cmi_hdl_model(hdl); 3355254Sgavinm s[2] = cmi_hdl_stepping(hdl); 3361414Scindi modid = modload_qualified(CPUMOD_SUBDIR, CPUMOD_PREFIX, 3375254Sgavinm cmi_hdl_vendorstr(hdl), ".", s, match, chosenp); 3381414Scindi 3391414Scindi if (modid == -1) 3401414Scindi return (NULL); 3411414Scindi 3421414Scindi modp = mod_hold_by_id(modid); 3431414Scindi cmi = cmi_load_modctl(modp); 3445254Sgavinm if (cmi) 3455254Sgavinm cmi_hold(cmi); 3461414Scindi mod_release_mod(modp); 3471414Scindi 3481414Scindi return (cmi); 3491414Scindi } 3501414Scindi 3511414Scindi /* 3525254Sgavinm * Try to load a cpu module with specific support for this chip type. 3531414Scindi */ 3545254Sgavinm static cmi_t * 3555254Sgavinm cmi_load_specific(cmi_hdl_t hdl, void **datap) 3561414Scindi { 3571414Scindi cmi_t *cmi; 3585254Sgavinm int err; 3595254Sgavinm int i; 3605254Sgavinm 3615254Sgavinm ASSERT(MUTEX_HELD(&cmi_load_lock)); 3625254Sgavinm 3635254Sgavinm for (i = CMI_MATCH_STEPPING; i >= CMI_MATCH_VENDOR; i--) { 3645254Sgavinm int suffixlevel; 3655254Sgavinm 3665254Sgavinm if ((cmi = cmi_load_module(hdl, i, &suffixlevel)) == NULL) 3675254Sgavinm return (NULL); 3685254Sgavinm 3695254Sgavinm /* 3705254Sgavinm * A module has loaded and has a _cmi_ops structure, and the 3715254Sgavinm * module has been held for this instance. Call its cmi_init 3725254Sgavinm * entry point - we expect success (0) or ENOTSUP. 3735254Sgavinm */ 3745254Sgavinm if ((err = cmi->cmi_ops->cmi_init(hdl, datap)) == 0) { 3755254Sgavinm if (boothowto & RB_VERBOSE) { 3765254Sgavinm printf("initialized cpu module '%s' on " 3775254Sgavinm "chip %d core %d strand %d\n", 3785254Sgavinm cmi->cmi_modp->mod_modname, 3795254Sgavinm cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl), 3805254Sgavinm cmi_hdl_strandid(hdl)); 3815254Sgavinm } 3825254Sgavinm return (cmi); 3835254Sgavinm } else if (err != ENOTSUP) { 3845254Sgavinm cmn_err(CE_WARN, "failed to init cpu module '%s' on " 3855254Sgavinm "chip %d core %d strand %d: err=%d\n", 3865254Sgavinm cmi->cmi_modp->mod_modname, 3875254Sgavinm cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl), 3885254Sgavinm cmi_hdl_strandid(hdl), err); 3895254Sgavinm } 3905254Sgavinm 3915254Sgavinm /* 3925254Sgavinm * The module failed or declined to init, so release 3935254Sgavinm * it and update i to be equal to the number 3945254Sgavinm * of suffices actually used in the last module path. 3955254Sgavinm */ 3965254Sgavinm cmi_rele(cmi); 3975254Sgavinm i = suffixlevel; 3985254Sgavinm } 3995254Sgavinm 4005254Sgavinm return (NULL); 4015254Sgavinm } 4025254Sgavinm 4035254Sgavinm /* 4045254Sgavinm * Load the generic IA32 MCA cpu module, which may still supplement 4055254Sgavinm * itself with model-specific support through cpu model-specific modules. 4065254Sgavinm */ 4075254Sgavinm static cmi_t * 4085254Sgavinm cmi_load_generic(cmi_hdl_t hdl, void **datap) 4095254Sgavinm { 4105254Sgavinm modctl_t *modp; 4115254Sgavinm cmi_t *cmi; 4125254Sgavinm int modid; 4135254Sgavinm int err; 4145254Sgavinm 4155254Sgavinm ASSERT(MUTEX_HELD(&cmi_load_lock)); 4165254Sgavinm 4175254Sgavinm if ((modid = modload(CPUMOD_SUBDIR, CPUMOD_PREFIX ".generic")) == -1) 4185254Sgavinm return (NULL); 4195254Sgavinm 4205254Sgavinm modp = mod_hold_by_id(modid); 4215254Sgavinm cmi = cmi_load_modctl(modp); 4225254Sgavinm if (cmi) 4235254Sgavinm cmi_hold(cmi); 4245254Sgavinm mod_release_mod(modp); 4255254Sgavinm 4265254Sgavinm if (cmi == NULL) 4275254Sgavinm return (NULL); 4285254Sgavinm 4295254Sgavinm if ((err = cmi->cmi_ops->cmi_init(hdl, datap)) != 0) { 4305254Sgavinm if (err != ENOTSUP) 4315254Sgavinm cmn_err(CE_WARN, CPUMOD_PREFIX ".generic failed to " 4325254Sgavinm "init: err=%d", err); 4335254Sgavinm cmi_rele(cmi); 4345254Sgavinm return (NULL); 4355254Sgavinm } 4365254Sgavinm 4375254Sgavinm return (cmi); 4385254Sgavinm } 4395254Sgavinm 4405254Sgavinm cmi_hdl_t 4415254Sgavinm cmi_init(enum cmi_hdl_class class, uint_t chipid, uint_t coreid, 442*7532SSean.Ye@Sun.COM uint_t strandid) 4435254Sgavinm { 4445254Sgavinm cmi_t *cmi = NULL; 4455254Sgavinm cmi_hdl_t hdl; 4461414Scindi void *data; 4471414Scindi 4485254Sgavinm if (cmi_no_init) { 4495254Sgavinm cmi_no_mca_init = 1; 4505254Sgavinm return (NULL); 4515254Sgavinm } 4525254Sgavinm 4531414Scindi mutex_enter(&cmi_load_lock); 4541414Scindi 455*7532SSean.Ye@Sun.COM if ((hdl = cmi_hdl_create(class, chipid, coreid, strandid)) == NULL) { 4561414Scindi mutex_exit(&cmi_load_lock); 4575254Sgavinm cmn_err(CE_WARN, "There will be no MCA support on chip %d " 4585254Sgavinm "core %d strand %d (cmi_hdl_create returned NULL)\n", 4595254Sgavinm chipid, coreid, strandid); 4605254Sgavinm return (NULL); 4611414Scindi } 4621414Scindi 4635254Sgavinm if (!cmi_force_generic) 4645254Sgavinm cmi = cmi_load_specific(hdl, &data); 4655254Sgavinm 4665254Sgavinm if (cmi == NULL && (cmi = cmi_load_generic(hdl, &data)) == NULL) { 4675254Sgavinm cmn_err(CE_WARN, "There will be no MCA support on chip %d " 4685254Sgavinm "core %d strand %d\n", chipid, coreid, strandid); 4695254Sgavinm cmi_hdl_rele(hdl); 4701414Scindi mutex_exit(&cmi_load_lock); 4715254Sgavinm return (NULL); 4721414Scindi } 4731414Scindi 4745254Sgavinm cmi_hdl_setcmi(hdl, cmi, data); 4751414Scindi 4765254Sgavinm cms_init(hdl); 4775254Sgavinm 4781414Scindi mutex_exit(&cmi_load_lock); 4791414Scindi 4805254Sgavinm return (hdl); 4811414Scindi } 4821414Scindi 4835254Sgavinm /* 4845254Sgavinm * cmi_fini is not called at the moment. It is intended to be called 4855254Sgavinm * on DR deconfigure of a cpu resource. It should not be called at 4865254Sgavinm * simple offline of a cpu. 4875254Sgavinm */ 4881414Scindi void 4895254Sgavinm cmi_fini(cmi_hdl_t hdl) 4901414Scindi { 4915254Sgavinm cmi_t *cmi = HDL2CMI(hdl); 4925254Sgavinm 4935254Sgavinm if (cms_present(hdl)) 4945254Sgavinm cms_fini(hdl); 4955254Sgavinm 4965254Sgavinm if (CMI_OP_PRESENT(cmi, cmi_fini)) 4975254Sgavinm CMI_OPS(cmi)->cmi_fini(hdl); 4985254Sgavinm 4995254Sgavinm cmi_hdl_rele(hdl); /* release hold obtained in cmi_hdl_create */ 5001414Scindi } 5011414Scindi 5025254Sgavinm /* 503*7532SSean.Ye@Sun.COM * cmi_post_startup is called from post_startup for the boot cpu only (no 504*7532SSean.Ye@Sun.COM * other cpus are started yet). 5055254Sgavinm */ 5061414Scindi void 5075254Sgavinm cmi_post_startup(void) 5081414Scindi { 5095254Sgavinm cmi_hdl_t hdl; 5105254Sgavinm cmi_t *cmi; 5115254Sgavinm 5125254Sgavinm if (cmi_no_mca_init != 0 || 5135254Sgavinm (hdl = cmi_hdl_any()) == NULL) /* short-term hold */ 5145254Sgavinm return; 5155254Sgavinm 5165254Sgavinm cmi = HDL2CMI(hdl); 5175254Sgavinm 5185254Sgavinm if (CMI_OP_PRESENT(cmi, cmi_post_startup)) 5195254Sgavinm CMI_OPS(cmi)->cmi_post_startup(hdl); 5205254Sgavinm 5215254Sgavinm cmi_hdl_rele(hdl); 5221414Scindi } 5231414Scindi 5242869Sgavinm /* 5252869Sgavinm * Called just once from start_other_cpus when all processors are started. 5262869Sgavinm * This will not be called for each cpu, so the registered op must not 527*7532SSean.Ye@Sun.COM * assume it is called as such. We are not necessarily executing on 528*7532SSean.Ye@Sun.COM * the boot cpu. 5292869Sgavinm */ 5301414Scindi void 5311642Sgavinm cmi_post_mpstartup(void) 5321642Sgavinm { 5335254Sgavinm cmi_hdl_t hdl; 5345254Sgavinm cmi_t *cmi; 5355254Sgavinm 5365254Sgavinm if (cmi_no_mca_init != 0 || 5375254Sgavinm (hdl = cmi_hdl_any()) == NULL) /* short-term hold */ 5385254Sgavinm return; 5395254Sgavinm 5405254Sgavinm cmi = HDL2CMI(hdl); 5415254Sgavinm 5425254Sgavinm if (CMI_OP_PRESENT(cmi, cmi_post_mpstartup)) 5435254Sgavinm CMI_OPS(cmi)->cmi_post_mpstartup(hdl); 5445254Sgavinm 5455254Sgavinm cmi_hdl_rele(hdl); 5461642Sgavinm } 5471642Sgavinm 5481642Sgavinm void 5495254Sgavinm cmi_faulted_enter(cmi_hdl_t hdl) 5501414Scindi { 5515254Sgavinm cmi_t *cmi = HDL2CMI(hdl); 5525254Sgavinm 5535254Sgavinm if (cmi_no_mca_init != 0) 5545254Sgavinm return; 5555254Sgavinm 5565254Sgavinm if (CMI_OP_PRESENT(cmi, cmi_faulted_enter)) 5575254Sgavinm CMI_OPS(cmi)->cmi_faulted_enter(hdl); 5585254Sgavinm } 5595254Sgavinm 5605254Sgavinm void 5615254Sgavinm cmi_faulted_exit(cmi_hdl_t hdl) 5625254Sgavinm { 5635254Sgavinm cmi_t *cmi = HDL2CMI(hdl); 5645254Sgavinm 5655254Sgavinm if (cmi_no_mca_init != 0) 5665254Sgavinm return; 5675254Sgavinm 5685254Sgavinm if (CMI_OP_PRESENT(cmi, cmi_faulted_exit)) 5695254Sgavinm CMI_OPS(cmi)->cmi_faulted_exit(hdl); 5701414Scindi } 5711414Scindi 5721414Scindi void 5735254Sgavinm cmi_mca_init(cmi_hdl_t hdl) 5741414Scindi { 5755254Sgavinm cmi_t *cmi; 5765254Sgavinm 5775254Sgavinm if (cmi_no_mca_init != 0) 5785254Sgavinm return; 5795254Sgavinm 5805254Sgavinm cmi = HDL2CMI(hdl); 5815254Sgavinm 5825254Sgavinm if (CMI_OP_PRESENT(cmi, cmi_mca_init)) 5835254Sgavinm CMI_OPS(cmi)->cmi_mca_init(hdl); 5841414Scindi } 5851414Scindi 5865254Sgavinm #define CMI_RESPONSE_PANIC 0x0 /* panic must have value 0 */ 5875254Sgavinm #define CMI_RESPONSE_NONE 0x1 5885254Sgavinm #define CMI_RESPONSE_CKILL 0x2 5895254Sgavinm #define CMI_RESPONSE_REBOOT 0x3 /* not implemented */ 5905254Sgavinm #define CMI_RESPONSE_ONTRAP_PROT 0x4 5915254Sgavinm #define CMI_RESPONSE_LOFAULT_PROT 0x5 5925254Sgavinm 5935254Sgavinm /* 5945254Sgavinm * Return 0 if we will panic in response to this machine check, otherwise 5955254Sgavinm * non-zero. If the caller is cmi_mca_trap in this file then the nonzero 5965254Sgavinm * return values are to be interpreted from CMI_RESPONSE_* above. 5975254Sgavinm * 5985254Sgavinm * This function must just return what will be done without actually 5995254Sgavinm * doing anything; this includes not changing the regs. 6005254Sgavinm */ 6011414Scindi int 6025254Sgavinm cmi_mce_response(struct regs *rp, uint64_t disp) 6031414Scindi { 6045254Sgavinm int panicrsp = cmi_panic_on_uncorrectable_error ? CMI_RESPONSE_PANIC : 6055254Sgavinm CMI_RESPONSE_NONE; 6065254Sgavinm on_trap_data_t *otp; 6075254Sgavinm 6085254Sgavinm ASSERT(rp != NULL); /* don't call for polling, only on #MC */ 6095254Sgavinm 6105254Sgavinm /* 6115254Sgavinm * If no bits are set in the disposition then there is nothing to 6125254Sgavinm * worry about and we do not need to trampoline to ontrap or 6135254Sgavinm * lofault handlers. 6145254Sgavinm */ 6155254Sgavinm if (disp == 0) 6165254Sgavinm return (CMI_RESPONSE_NONE); 6175254Sgavinm 6185254Sgavinm /* 6195254Sgavinm * Unconstrained errors cannot be forgiven, even by ontrap or 6205254Sgavinm * lofault protection. The data is not poisoned and may not 6215254Sgavinm * even belong to the trapped context - eg a writeback of 6225254Sgavinm * data that is found to be bad. 6235254Sgavinm */ 6245254Sgavinm if (disp & CMI_ERRDISP_UC_UNCONSTRAINED) 6255254Sgavinm return (panicrsp); 6265254Sgavinm 6275254Sgavinm /* 6285254Sgavinm * ontrap OT_DATA_EC and lofault protection forgive any disposition 6295254Sgavinm * other than unconstrained, even those normally forced fatal. 6305254Sgavinm */ 6315254Sgavinm if ((otp = curthread->t_ontrap) != NULL && otp->ot_prot & OT_DATA_EC) 6325254Sgavinm return (CMI_RESPONSE_ONTRAP_PROT); 6335254Sgavinm else if (curthread->t_lofault) 6345254Sgavinm return (CMI_RESPONSE_LOFAULT_PROT); 6355254Sgavinm 6365254Sgavinm /* 6375254Sgavinm * Forced-fatal errors are terminal even in user mode. 6385254Sgavinm */ 6395254Sgavinm if (disp & CMI_ERRDISP_FORCEFATAL) 6405254Sgavinm return (panicrsp); 6415254Sgavinm 6425254Sgavinm /* 6435254Sgavinm * If the trapped context is corrupt or we have no instruction pointer 6445254Sgavinm * to resume at (and aren't trampolining to a fault handler) 6455254Sgavinm * then in the kernel case we must panic and in usermode we 6465254Sgavinm * kill the affected contract. 6475254Sgavinm */ 6485254Sgavinm if (disp & (CMI_ERRDISP_CURCTXBAD | CMI_ERRDISP_RIPV_INVALID)) 6495254Sgavinm return (USERMODE(rp->r_cs) ? CMI_RESPONSE_CKILL : panicrsp); 6505254Sgavinm 6515254Sgavinm /* 6525254Sgavinm * Anything else is harmless 6535254Sgavinm */ 6545254Sgavinm return (CMI_RESPONSE_NONE); 6551414Scindi } 6561414Scindi 6575254Sgavinm int cma_mca_trap_panic_suppressed = 0; 6585254Sgavinm 6595254Sgavinm static void 6605254Sgavinm cmi_mca_panic(void) 6611414Scindi { 6625254Sgavinm if (cmi_panic_on_uncorrectable_error) { 6635254Sgavinm fm_panic("Unrecoverable Machine-Check Exception"); 6645254Sgavinm } else { 6655254Sgavinm cmn_err(CE_WARN, "suppressing panic from fatal #mc"); 6665254Sgavinm cma_mca_trap_panic_suppressed++; 6675254Sgavinm } 6681414Scindi } 6691414Scindi 6705254Sgavinm 6715254Sgavinm int cma_mca_trap_contract_kills = 0; 6725254Sgavinm int cma_mca_trap_ontrap_forgiven = 0; 6735254Sgavinm int cma_mca_trap_lofault_forgiven = 0; 6745254Sgavinm 6755254Sgavinm /* 6765254Sgavinm * Native #MC handler - we branch to here from mcetrap 6775254Sgavinm */ 6785254Sgavinm /*ARGSUSED*/ 6791414Scindi void 6801414Scindi cmi_mca_trap(struct regs *rp) 6811414Scindi { 6825254Sgavinm #ifndef __xpv 6835254Sgavinm cmi_hdl_t hdl = NULL; 6845254Sgavinm uint64_t disp; 6855254Sgavinm cmi_t *cmi; 6865254Sgavinm int s; 6875254Sgavinm 6885254Sgavinm if (cmi_no_mca_init != 0) 6895254Sgavinm return; 6905254Sgavinm 6915254Sgavinm /* 6925254Sgavinm * This function can call cmn_err, and the cpu module cmi_mca_trap 6935254Sgavinm * entry point may also elect to call cmn_err (e.g., if it can't 6945254Sgavinm * log the error onto an errorq, say very early in boot). 6955254Sgavinm * We need to let cprintf know that we must not block. 6965254Sgavinm */ 6975254Sgavinm s = spl8(); 6985254Sgavinm 6995254Sgavinm if ((hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU), 7005254Sgavinm cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) == NULL || 7015254Sgavinm (cmi = HDL2CMI(hdl)) == NULL || 7025254Sgavinm !CMI_OP_PRESENT(cmi, cmi_mca_trap)) { 7035254Sgavinm 7045254Sgavinm cmn_err(CE_WARN, "#MC exception on cpuid %d: %s", 7055254Sgavinm CPU->cpu_id, 7065254Sgavinm hdl ? "handle lookup ok but no #MC handler found" : 7075254Sgavinm "handle lookup failed"); 7085254Sgavinm 7095254Sgavinm if (hdl != NULL) 7105254Sgavinm cmi_hdl_rele(hdl); 7115254Sgavinm 7125254Sgavinm splx(s); 7135254Sgavinm return; 7141414Scindi } 7155254Sgavinm 7165254Sgavinm disp = CMI_OPS(cmi)->cmi_mca_trap(hdl, rp); 7171414Scindi 7185254Sgavinm switch (cmi_mce_response(rp, disp)) { 7195254Sgavinm default: 7205254Sgavinm cmn_err(CE_WARN, "Invalid response from cmi_mce_response"); 7215254Sgavinm /*FALLTHRU*/ 7225254Sgavinm 7235254Sgavinm case CMI_RESPONSE_PANIC: 7245254Sgavinm cmi_mca_panic(); 7255254Sgavinm break; 7265254Sgavinm 7275254Sgavinm case CMI_RESPONSE_NONE: 7285254Sgavinm break; 7295254Sgavinm 7305254Sgavinm case CMI_RESPONSE_CKILL: 7315254Sgavinm ttolwp(curthread)->lwp_pcb.pcb_flags |= ASYNC_HWERR; 7325254Sgavinm aston(curthread); 7335254Sgavinm cma_mca_trap_contract_kills++; 7345254Sgavinm break; 7351414Scindi 7365254Sgavinm case CMI_RESPONSE_ONTRAP_PROT: { 7375254Sgavinm on_trap_data_t *otp = curthread->t_ontrap; 7385254Sgavinm otp->ot_trap = OT_DATA_EC; 7395254Sgavinm rp->r_pc = otp->ot_trampoline; 7405254Sgavinm cma_mca_trap_ontrap_forgiven++; 7415254Sgavinm break; 7425254Sgavinm } 7431414Scindi 7445254Sgavinm case CMI_RESPONSE_LOFAULT_PROT: 7455254Sgavinm rp->r_r0 = EFAULT; 7465254Sgavinm rp->r_pc = curthread->t_lofault; 7475254Sgavinm cma_mca_trap_lofault_forgiven++; 7485254Sgavinm break; 7495254Sgavinm } 7505254Sgavinm 7515254Sgavinm cmi_hdl_rele(hdl); 7525254Sgavinm splx(s); 7535254Sgavinm #endif /* __xpv */ 7541414Scindi } 7551414Scindi 7561414Scindi void 7575254Sgavinm cmi_hdl_poke(cmi_hdl_t hdl) 7581414Scindi { 7595254Sgavinm cmi_t *cmi = HDL2CMI(hdl); 7605254Sgavinm 7615254Sgavinm if (!CMI_OP_PRESENT(cmi, cmi_hdl_poke)) 7625254Sgavinm return; 7635254Sgavinm 7645254Sgavinm CMI_OPS(cmi)->cmi_hdl_poke(hdl); 7651414Scindi } 7661414Scindi 767*7532SSean.Ye@Sun.COM #ifndef __xpv 7681414Scindi void 7697349SAdrian.Frost@Sun.COM cmi_cmci_trap() 7707349SAdrian.Frost@Sun.COM { 7717349SAdrian.Frost@Sun.COM cmi_hdl_t hdl = NULL; 7727349SAdrian.Frost@Sun.COM cmi_t *cmi; 7737349SAdrian.Frost@Sun.COM 7747349SAdrian.Frost@Sun.COM if (cmi_no_mca_init != 0) 7757349SAdrian.Frost@Sun.COM return; 7767349SAdrian.Frost@Sun.COM 7777349SAdrian.Frost@Sun.COM if ((hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU), 7787349SAdrian.Frost@Sun.COM cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) == NULL || 7797349SAdrian.Frost@Sun.COM (cmi = HDL2CMI(hdl)) == NULL || 7807349SAdrian.Frost@Sun.COM !CMI_OP_PRESENT(cmi, cmi_cmci_trap)) { 7817349SAdrian.Frost@Sun.COM 7827349SAdrian.Frost@Sun.COM cmn_err(CE_WARN, "CMCI interrupt on cpuid %d: %s", 7837349SAdrian.Frost@Sun.COM CPU->cpu_id, 7847349SAdrian.Frost@Sun.COM hdl ? "handle lookup ok but no CMCI handler found" : 7857349SAdrian.Frost@Sun.COM "handle lookup failed"); 7867349SAdrian.Frost@Sun.COM 7877349SAdrian.Frost@Sun.COM if (hdl != NULL) 7887349SAdrian.Frost@Sun.COM cmi_hdl_rele(hdl); 7897349SAdrian.Frost@Sun.COM 7907349SAdrian.Frost@Sun.COM return; 7917349SAdrian.Frost@Sun.COM } 7927349SAdrian.Frost@Sun.COM 7937349SAdrian.Frost@Sun.COM CMI_OPS(cmi)->cmi_cmci_trap(hdl); 7947349SAdrian.Frost@Sun.COM 7957349SAdrian.Frost@Sun.COM cmi_hdl_rele(hdl); 796*7532SSean.Ye@Sun.COM } 7977349SAdrian.Frost@Sun.COM #endif /* __xpv */ 7987349SAdrian.Frost@Sun.COM 7997349SAdrian.Frost@Sun.COM void 8005254Sgavinm cmi_mc_register(cmi_hdl_t hdl, const cmi_mc_ops_t *mcops, void *mcdata) 8011414Scindi { 8025254Sgavinm if (!cmi_no_mca_init) 8035254Sgavinm cmi_hdl_setmc(hdl, mcops, mcdata); 8041414Scindi } 8051414Scindi 806*7532SSean.Ye@Sun.COM void 807*7532SSean.Ye@Sun.COM cmi_mc_sw_memscrub_disable(void) 808*7532SSean.Ye@Sun.COM { 809*7532SSean.Ye@Sun.COM memscrub_disable(); 810*7532SSean.Ye@Sun.COM } 811*7532SSean.Ye@Sun.COM 8125254Sgavinm cmi_errno_t 8133164Sgavinm cmi_mc_patounum(uint64_t pa, uint8_t valid_hi, uint8_t valid_lo, uint32_t synd, 8143164Sgavinm int syndtype, mc_unum_t *up) 8151414Scindi { 8161414Scindi const struct cmi_mc_ops *mcops; 8175254Sgavinm cmi_hdl_t hdl; 8185254Sgavinm cmi_errno_t rv; 8195254Sgavinm 8205254Sgavinm if (cmi_no_mca_init || 8215254Sgavinm (hdl = cmi_hdl_any()) == NULL) /* short-term hold */ 8225254Sgavinm return (CMIERR_MC_ABSENT); 8231414Scindi 8245254Sgavinm if ((mcops = cmi_hdl_getmcops(hdl)) == NULL || 8255254Sgavinm mcops->cmi_mc_patounum == NULL) { 8265254Sgavinm cmi_hdl_rele(hdl); 8275254Sgavinm return (CMIERR_MC_NOTSUP); 8285254Sgavinm } 8291414Scindi 8305254Sgavinm rv = mcops->cmi_mc_patounum(cmi_hdl_getmcdata(hdl), pa, valid_hi, 8315254Sgavinm valid_lo, synd, syndtype, up); 8325254Sgavinm 8335254Sgavinm cmi_hdl_rele(hdl); 8345254Sgavinm 8355254Sgavinm return (rv); 8361414Scindi } 8371414Scindi 8385254Sgavinm cmi_errno_t 8391414Scindi cmi_mc_unumtopa(mc_unum_t *up, nvlist_t *nvl, uint64_t *pap) 8401414Scindi { 8411414Scindi const struct cmi_mc_ops *mcops; 8425254Sgavinm cmi_hdl_t hdl; 8435254Sgavinm cmi_errno_t rv; 844*7532SSean.Ye@Sun.COM nvlist_t *hcsp; 8451414Scindi 8461414Scindi if (up != NULL && nvl != NULL) 8475254Sgavinm return (CMIERR_API); /* convert from just one form */ 8485254Sgavinm 8495254Sgavinm if (cmi_no_mca_init || 8505254Sgavinm (hdl = cmi_hdl_any()) == NULL) /* short-term hold */ 8515254Sgavinm return (CMIERR_MC_ABSENT); 8525254Sgavinm 8535254Sgavinm if ((mcops = cmi_hdl_getmcops(hdl)) == NULL || 8545254Sgavinm mcops->cmi_mc_unumtopa == NULL) { 8555254Sgavinm cmi_hdl_rele(hdl); 8565254Sgavinm 857*7532SSean.Ye@Sun.COM if (nvl != NULL && nvlist_lookup_nvlist(nvl, 858*7532SSean.Ye@Sun.COM FM_FMRI_HC_SPECIFIC, &hcsp) == 0 && 859*7532SSean.Ye@Sun.COM (nvlist_lookup_uint64(hcsp, 860*7532SSean.Ye@Sun.COM "asru-" FM_FMRI_HC_SPECIFIC_PHYSADDR, pap) == 0 || 861*7532SSean.Ye@Sun.COM nvlist_lookup_uint64(hcsp, FM_FMRI_HC_SPECIFIC_PHYSADDR, 862*7532SSean.Ye@Sun.COM pap) == 0)) { 8635254Sgavinm return (CMIERR_MC_PARTIALUNUMTOPA); 8645254Sgavinm } else { 8655254Sgavinm return (mcops && mcops->cmi_mc_unumtopa ? 8665254Sgavinm CMIERR_MC_NOTSUP : CMIERR_MC_ABSENT); 8675254Sgavinm } 8685254Sgavinm } 8695254Sgavinm 8705254Sgavinm rv = mcops->cmi_mc_unumtopa(cmi_hdl_getmcdata(hdl), up, nvl, pap); 8715254Sgavinm 8725254Sgavinm cmi_hdl_rele(hdl); 8735254Sgavinm 8745254Sgavinm return (rv); 8755254Sgavinm } 8761414Scindi 8775254Sgavinm void 8785254Sgavinm cmi_mc_logout(cmi_hdl_t hdl, boolean_t ismc, boolean_t sync) 8795254Sgavinm { 8805254Sgavinm const struct cmi_mc_ops *mcops; 8815254Sgavinm 8825254Sgavinm if (cmi_no_mca_init || (mcops = cmi_hdl_getmcops(hdl)) == NULL) 8835254Sgavinm return; 8845254Sgavinm 8855254Sgavinm if (mcops->cmi_mc_logout != NULL) 8865254Sgavinm mcops->cmi_mc_logout(hdl, ismc, sync); 8875254Sgavinm } 8881414Scindi 8895254Sgavinm cmi_errno_t 8905254Sgavinm cmi_hdl_msrinject(cmi_hdl_t hdl, cmi_mca_regs_t *regs, uint_t nregs, 8915254Sgavinm int force) 8925254Sgavinm { 8935254Sgavinm cmi_t *cmi = cmi_hdl_getcmi(hdl); 894*7532SSean.Ye@Sun.COM cmi_errno_t rc; 8955254Sgavinm 8965254Sgavinm if (!CMI_OP_PRESENT(cmi, cmi_msrinject)) 8975254Sgavinm return (CMIERR_NOTSUP); 8985254Sgavinm 899*7532SSean.Ye@Sun.COM cmi_hdl_inj_begin(hdl); 900*7532SSean.Ye@Sun.COM rc = CMI_OPS(cmi)->cmi_msrinject(hdl, regs, nregs, force); 901*7532SSean.Ye@Sun.COM cmi_hdl_inj_end(hdl); 902*7532SSean.Ye@Sun.COM 903*7532SSean.Ye@Sun.COM return (rc); 9041414Scindi } 9055254Sgavinm 9065254Sgavinm boolean_t 9075254Sgavinm cmi_panic_on_ue(void) 9085254Sgavinm { 9095254Sgavinm return (cmi_panic_on_uncorrectable_error ? B_TRUE : B_FALSE); 9105254Sgavinm } 911*7532SSean.Ye@Sun.COM 912*7532SSean.Ye@Sun.COM void 913*7532SSean.Ye@Sun.COM cmi_panic_callback(void) 914*7532SSean.Ye@Sun.COM { 915*7532SSean.Ye@Sun.COM cmi_hdl_t hdl; 916*7532SSean.Ye@Sun.COM cmi_t *cmi; 917*7532SSean.Ye@Sun.COM 918*7532SSean.Ye@Sun.COM if (cmi_no_mca_init || (hdl = cmi_hdl_any()) == NULL) 919*7532SSean.Ye@Sun.COM return; 920*7532SSean.Ye@Sun.COM 921*7532SSean.Ye@Sun.COM cmi = cmi_hdl_getcmi(hdl); 922*7532SSean.Ye@Sun.COM if (CMI_OP_PRESENT(cmi, cmi_panic_callback)) 923*7532SSean.Ye@Sun.COM CMI_OPS(cmi)->cmi_panic_callback(); 924*7532SSean.Ye@Sun.COM 925*7532SSean.Ye@Sun.COM cmi_hdl_rele(hdl); 926*7532SSean.Ye@Sun.COM } 927