1*0Sstevel@tonic-gate /* 2*0Sstevel@tonic-gate * CDDL HEADER START 3*0Sstevel@tonic-gate * 4*0Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*0Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*0Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*0Sstevel@tonic-gate * with the License. 8*0Sstevel@tonic-gate * 9*0Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*0Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*0Sstevel@tonic-gate * See the License for the specific language governing permissions 12*0Sstevel@tonic-gate * and limitations under the License. 13*0Sstevel@tonic-gate * 14*0Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*0Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*0Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*0Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*0Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*0Sstevel@tonic-gate * 20*0Sstevel@tonic-gate * CDDL HEADER END 21*0Sstevel@tonic-gate */ 22*0Sstevel@tonic-gate /* 23*0Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24*0Sstevel@tonic-gate * Use is subject to license terms. 25*0Sstevel@tonic-gate */ 26*0Sstevel@tonic-gate 27*0Sstevel@tonic-gate /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */ 28*0Sstevel@tonic-gate /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */ 29*0Sstevel@tonic-gate /* All Rights Reserved */ 30*0Sstevel@tonic-gate 31*0Sstevel@tonic-gate /* Copyright (c) 1987, 1988 Microsoft Corporation */ 32*0Sstevel@tonic-gate /* All Rights Reserved */ 33*0Sstevel@tonic-gate 34*0Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 35*0Sstevel@tonic-gate 36*0Sstevel@tonic-gate #include <sys/types.h> 37*0Sstevel@tonic-gate #include <sys/param.h> 38*0Sstevel@tonic-gate #include <sys/signal.h> 39*0Sstevel@tonic-gate #include <sys/regset.h> 40*0Sstevel@tonic-gate #include <sys/privregs.h> 41*0Sstevel@tonic-gate #include <sys/psw.h> 42*0Sstevel@tonic-gate #include <sys/trap.h> 43*0Sstevel@tonic-gate #include <sys/fault.h> 44*0Sstevel@tonic-gate #include <sys/systm.h> 45*0Sstevel@tonic-gate #include <sys/user.h> 46*0Sstevel@tonic-gate #include <sys/file.h> 47*0Sstevel@tonic-gate #include <sys/proc.h> 48*0Sstevel@tonic-gate #include <sys/pcb.h> 49*0Sstevel@tonic-gate #include <sys/lwp.h> 50*0Sstevel@tonic-gate #include <sys/cpuvar.h> 51*0Sstevel@tonic-gate #include <sys/thread.h> 52*0Sstevel@tonic-gate #include <sys/disp.h> 53*0Sstevel@tonic-gate #include <sys/fp.h> 54*0Sstevel@tonic-gate #include <sys/siginfo.h> 55*0Sstevel@tonic-gate #include <sys/archsystm.h> 56*0Sstevel@tonic-gate #include <sys/kmem.h> 57*0Sstevel@tonic-gate #include <sys/debug.h> 58*0Sstevel@tonic-gate #include <sys/x86_archext.h> 59*0Sstevel@tonic-gate #include <sys/sysmacros.h> 60*0Sstevel@tonic-gate 61*0Sstevel@tonic-gate /*CSTYLED*/ 62*0Sstevel@tonic-gate #pragma align 16 (sse_initial) 63*0Sstevel@tonic-gate 64*0Sstevel@tonic-gate /* 65*0Sstevel@tonic-gate * Initial kfpu state for SSE/SSE2 used by fpinit() 66*0Sstevel@tonic-gate */ 67*0Sstevel@tonic-gate const struct fxsave_state sse_initial = { 68*0Sstevel@tonic-gate FPU_CW_INIT, /* fx_fcw */ 69*0Sstevel@tonic-gate 0, /* fx_fsw */ 70*0Sstevel@tonic-gate 0, /* fx_fctw */ 71*0Sstevel@tonic-gate 0, /* fx_fop */ 72*0Sstevel@tonic-gate #if defined(__amd64) 73*0Sstevel@tonic-gate 0, /* fx_rip */ 74*0Sstevel@tonic-gate 0, /* fx_rdp */ 75*0Sstevel@tonic-gate #else 76*0Sstevel@tonic-gate 0, /* fx_eip */ 77*0Sstevel@tonic-gate 0, /* fx_cs */ 78*0Sstevel@tonic-gate 0, /* __fx_ign0 */ 79*0Sstevel@tonic-gate 0, /* fx_dp */ 80*0Sstevel@tonic-gate 0, /* fx_ds */ 81*0Sstevel@tonic-gate 0, /* __fx_ign1 */ 82*0Sstevel@tonic-gate #endif /* __amd64 */ 83*0Sstevel@tonic-gate SSE_MXCSR_INIT /* fx_mxcsr */ 84*0Sstevel@tonic-gate /* rest of structure is zero */ 85*0Sstevel@tonic-gate }; 86*0Sstevel@tonic-gate 87*0Sstevel@tonic-gate /* 88*0Sstevel@tonic-gate * mxcsr_mask value (possibly reset in fpu_probe); used to avoid 89*0Sstevel@tonic-gate * the #gp exception caused by setting unsupported bits in the 90*0Sstevel@tonic-gate * MXCSR register 91*0Sstevel@tonic-gate */ 92*0Sstevel@tonic-gate uint32_t sse_mxcsr_mask = SSE_MXCSR_MASK_DEFAULT; 93*0Sstevel@tonic-gate 94*0Sstevel@tonic-gate /* 95*0Sstevel@tonic-gate * Initial kfpu state for x87 used by fpinit() 96*0Sstevel@tonic-gate */ 97*0Sstevel@tonic-gate const struct fnsave_state x87_initial = { 98*0Sstevel@tonic-gate FPU_CW_INIT, /* f_fcw */ 99*0Sstevel@tonic-gate 0, /* __f_ign0 */ 100*0Sstevel@tonic-gate 0, /* f_fsw */ 101*0Sstevel@tonic-gate 0, /* __f_ign1 */ 102*0Sstevel@tonic-gate 0xffff, /* f_ftw */ 103*0Sstevel@tonic-gate /* rest of structure is zero */ 104*0Sstevel@tonic-gate }; 105*0Sstevel@tonic-gate 106*0Sstevel@tonic-gate #if defined(__amd64) 107*0Sstevel@tonic-gate #define fpsave_begin fpxsave_begin 108*0Sstevel@tonic-gate #elif defined(__i386) 109*0Sstevel@tonic-gate /* 110*0Sstevel@tonic-gate * This vector is patched to fpxsave_begin() if we discover 111*0Sstevel@tonic-gate * we have an SSE-capable chip in fpu_probe(). 112*0Sstevel@tonic-gate */ 113*0Sstevel@tonic-gate void (*fpsave_begin)(void *) = fpnsave_begin; 114*0Sstevel@tonic-gate #endif 115*0Sstevel@tonic-gate 116*0Sstevel@tonic-gate static int fpe_sicode(uint_t); 117*0Sstevel@tonic-gate static int fpe_simd_sicode(uint_t); 118*0Sstevel@tonic-gate 119*0Sstevel@tonic-gate /* 120*0Sstevel@tonic-gate * Copy the state of parent lwp's floating point context into the new lwp. 121*0Sstevel@tonic-gate * Invoked for both fork() and lwp_create(). 122*0Sstevel@tonic-gate * 123*0Sstevel@tonic-gate * Note that we inherit -only- the control state (e.g. exception masks, 124*0Sstevel@tonic-gate * rounding, precision control, etc.); the FPU registers are otherwise 125*0Sstevel@tonic-gate * reset to their initial state. 126*0Sstevel@tonic-gate */ 127*0Sstevel@tonic-gate static void 128*0Sstevel@tonic-gate fp_new_lwp(kthread_id_t t, kthread_id_t ct) 129*0Sstevel@tonic-gate { 130*0Sstevel@tonic-gate struct fpu_ctx *fp; /* parent fpu context */ 131*0Sstevel@tonic-gate struct fpu_ctx *cfp; /* new fpu context */ 132*0Sstevel@tonic-gate struct fxsave_state *fx, *cfx; 133*0Sstevel@tonic-gate 134*0Sstevel@tonic-gate ASSERT(fp_kind != FP_NO); 135*0Sstevel@tonic-gate 136*0Sstevel@tonic-gate fp = &t->t_lwp->lwp_pcb.pcb_fpu; 137*0Sstevel@tonic-gate cfp = &ct->t_lwp->lwp_pcb.pcb_fpu; 138*0Sstevel@tonic-gate 139*0Sstevel@tonic-gate /* 140*0Sstevel@tonic-gate * If the parent FPU state is still in the FPU hw then save it; 141*0Sstevel@tonic-gate * conveniently, fp_save() already does this for us nicely. 142*0Sstevel@tonic-gate */ 143*0Sstevel@tonic-gate fp_save(fp); 144*0Sstevel@tonic-gate 145*0Sstevel@tonic-gate cfp->fpu_flags = FPU_EN | FPU_VALID; 146*0Sstevel@tonic-gate cfp->fpu_regs.kfpu_status = 0; 147*0Sstevel@tonic-gate cfp->fpu_regs.kfpu_xstatus = 0; 148*0Sstevel@tonic-gate 149*0Sstevel@tonic-gate #if defined(__amd64) 150*0Sstevel@tonic-gate fx = &fp->fpu_regs.kfpu_u.kfpu_fx; 151*0Sstevel@tonic-gate cfx = &cfp->fpu_regs.kfpu_u.kfpu_fx; 152*0Sstevel@tonic-gate bcopy(&sse_initial, cfx, sizeof (*cfx)); 153*0Sstevel@tonic-gate cfx->fx_mxcsr = fx->fx_mxcsr & ~SSE_MXCSR_EFLAGS; 154*0Sstevel@tonic-gate cfx->fx_fcw = fx->fx_fcw; 155*0Sstevel@tonic-gate #else 156*0Sstevel@tonic-gate if (fp_kind == __FP_SSE) { 157*0Sstevel@tonic-gate fx = &fp->fpu_regs.kfpu_u.kfpu_fx; 158*0Sstevel@tonic-gate cfx = &cfp->fpu_regs.kfpu_u.kfpu_fx; 159*0Sstevel@tonic-gate bcopy(&sse_initial, cfx, sizeof (*cfx)); 160*0Sstevel@tonic-gate cfx->fx_mxcsr = fx->fx_mxcsr & ~SSE_MXCSR_EFLAGS; 161*0Sstevel@tonic-gate cfx->fx_fcw = fx->fx_fcw; 162*0Sstevel@tonic-gate } else { 163*0Sstevel@tonic-gate struct fnsave_state *fn = &fp->fpu_regs.kfpu_u.kfpu_fn; 164*0Sstevel@tonic-gate struct fnsave_state *cfn = &cfp->fpu_regs.kfpu_u.kfpu_fn; 165*0Sstevel@tonic-gate 166*0Sstevel@tonic-gate bcopy(&x87_initial, cfn, sizeof (*cfn)); 167*0Sstevel@tonic-gate cfn->f_fcw = fn->f_fcw; 168*0Sstevel@tonic-gate } 169*0Sstevel@tonic-gate #endif 170*0Sstevel@tonic-gate installctx(ct, cfp, 171*0Sstevel@tonic-gate fpsave_begin, NULL, fp_new_lwp, fp_new_lwp, NULL, fp_free); 172*0Sstevel@tonic-gate /* 173*0Sstevel@tonic-gate * Now, when the new lwp starts running, it will take a trap 174*0Sstevel@tonic-gate * that will be handled inline in the trap table to cause 175*0Sstevel@tonic-gate * the appropriate f*rstor instruction to load the save area we 176*0Sstevel@tonic-gate * constructed above directly into the hardware. 177*0Sstevel@tonic-gate */ 178*0Sstevel@tonic-gate } 179*0Sstevel@tonic-gate 180*0Sstevel@tonic-gate /* 181*0Sstevel@tonic-gate * Free any state associated with floating point context. 182*0Sstevel@tonic-gate * Fp_free can be called in three cases: 183*0Sstevel@tonic-gate * 1) from reaper -> thread_free -> ctxfree -> fp_free 184*0Sstevel@tonic-gate * fp context belongs to a thread on deathrow 185*0Sstevel@tonic-gate * nothing to do, thread will never be resumed 186*0Sstevel@tonic-gate * thread calling ctxfree is reaper 187*0Sstevel@tonic-gate * 188*0Sstevel@tonic-gate * 2) from exec -> ctxfree -> fp_free 189*0Sstevel@tonic-gate * fp context belongs to the current thread 190*0Sstevel@tonic-gate * must disable fpu, thread calling ctxfree is curthread 191*0Sstevel@tonic-gate * 192*0Sstevel@tonic-gate * 3) from restorecontext -> setfpregs -> fp_free 193*0Sstevel@tonic-gate * we have a modified context in the memory (lwp->pcb_fpu) 194*0Sstevel@tonic-gate * disable fpu and release the fp context for the CPU 195*0Sstevel@tonic-gate * 196*0Sstevel@tonic-gate */ 197*0Sstevel@tonic-gate /*ARGSUSED*/ 198*0Sstevel@tonic-gate void 199*0Sstevel@tonic-gate fp_free(struct fpu_ctx *fp, int isexec) 200*0Sstevel@tonic-gate { 201*0Sstevel@tonic-gate ASSERT(fp_kind != FP_NO); 202*0Sstevel@tonic-gate 203*0Sstevel@tonic-gate if (fp->fpu_flags & FPU_VALID) 204*0Sstevel@tonic-gate return; 205*0Sstevel@tonic-gate 206*0Sstevel@tonic-gate kpreempt_disable(); 207*0Sstevel@tonic-gate /* 208*0Sstevel@tonic-gate * We want to do fpsave rather than fpdisable so that we can 209*0Sstevel@tonic-gate * keep the fpu_flags as FPU_VALID tracking the CR0_TS bit 210*0Sstevel@tonic-gate */ 211*0Sstevel@tonic-gate fp->fpu_flags |= FPU_VALID; 212*0Sstevel@tonic-gate /* If for current thread disable FP to track FPU_VALID */ 213*0Sstevel@tonic-gate if (curthread->t_lwp && fp == &curthread->t_lwp->lwp_pcb.pcb_fpu) { 214*0Sstevel@tonic-gate /* Clear errors if any to prevent frstor from complaining */ 215*0Sstevel@tonic-gate (void) fperr_reset(); 216*0Sstevel@tonic-gate if (fp_kind == __FP_SSE) 217*0Sstevel@tonic-gate (void) fpxerr_reset(); 218*0Sstevel@tonic-gate fpdisable(); 219*0Sstevel@tonic-gate } 220*0Sstevel@tonic-gate kpreempt_enable(); 221*0Sstevel@tonic-gate } 222*0Sstevel@tonic-gate 223*0Sstevel@tonic-gate /* 224*0Sstevel@tonic-gate * Store the floating point state and disable the floating point unit. 225*0Sstevel@tonic-gate */ 226*0Sstevel@tonic-gate void 227*0Sstevel@tonic-gate fp_save(struct fpu_ctx *fp) 228*0Sstevel@tonic-gate { 229*0Sstevel@tonic-gate ASSERT(fp_kind != FP_NO); 230*0Sstevel@tonic-gate 231*0Sstevel@tonic-gate kpreempt_disable(); 232*0Sstevel@tonic-gate if (!fp || fp->fpu_flags & FPU_VALID) { 233*0Sstevel@tonic-gate kpreempt_enable(); 234*0Sstevel@tonic-gate return; 235*0Sstevel@tonic-gate } 236*0Sstevel@tonic-gate ASSERT(curthread->t_lwp && fp == &curthread->t_lwp->lwp_pcb.pcb_fpu); 237*0Sstevel@tonic-gate 238*0Sstevel@tonic-gate #if defined(__amd64) 239*0Sstevel@tonic-gate fpxsave(&fp->fpu_regs.kfpu_u.kfpu_fx); 240*0Sstevel@tonic-gate #else 241*0Sstevel@tonic-gate switch (fp_kind) { 242*0Sstevel@tonic-gate case __FP_SSE: 243*0Sstevel@tonic-gate fpxsave(&fp->fpu_regs.kfpu_u.kfpu_fx); 244*0Sstevel@tonic-gate break; 245*0Sstevel@tonic-gate default: 246*0Sstevel@tonic-gate fpsave(&fp->fpu_regs.kfpu_u.kfpu_fn); 247*0Sstevel@tonic-gate break; 248*0Sstevel@tonic-gate } 249*0Sstevel@tonic-gate #endif 250*0Sstevel@tonic-gate fp->fpu_flags |= FPU_VALID; 251*0Sstevel@tonic-gate kpreempt_enable(); 252*0Sstevel@tonic-gate } 253*0Sstevel@tonic-gate 254*0Sstevel@tonic-gate /* 255*0Sstevel@tonic-gate * Restore the FPU context for the thread: 256*0Sstevel@tonic-gate * The possibilities are: 257*0Sstevel@tonic-gate * 1. No active FPU context: Load the new context into the FPU hw 258*0Sstevel@tonic-gate * and enable the FPU. 259*0Sstevel@tonic-gate */ 260*0Sstevel@tonic-gate void 261*0Sstevel@tonic-gate fp_restore(struct fpu_ctx *fp) 262*0Sstevel@tonic-gate { 263*0Sstevel@tonic-gate #if defined(__amd64) 264*0Sstevel@tonic-gate fpxrestore(&fp->fpu_regs.kfpu_u.kfpu_fx); 265*0Sstevel@tonic-gate #else 266*0Sstevel@tonic-gate /* case 2 */ 267*0Sstevel@tonic-gate if (fp_kind == __FP_SSE) 268*0Sstevel@tonic-gate fpxrestore(&fp->fpu_regs.kfpu_u.kfpu_fx); 269*0Sstevel@tonic-gate else 270*0Sstevel@tonic-gate fprestore(&fp->fpu_regs.kfpu_u.kfpu_fn); 271*0Sstevel@tonic-gate #endif 272*0Sstevel@tonic-gate fp->fpu_flags &= ~FPU_VALID; 273*0Sstevel@tonic-gate } 274*0Sstevel@tonic-gate 275*0Sstevel@tonic-gate 276*0Sstevel@tonic-gate /* 277*0Sstevel@tonic-gate * Seeds the initial state for the current thread. The possibilities are: 278*0Sstevel@tonic-gate * 1. Another process has modified the FPU state before we have done any 279*0Sstevel@tonic-gate * initialization: Load the FPU state from the LWP state. 280*0Sstevel@tonic-gate * 2. The FPU state has not been externally modified: Load a clean state. 281*0Sstevel@tonic-gate */ 282*0Sstevel@tonic-gate static void 283*0Sstevel@tonic-gate fp_seed(void) 284*0Sstevel@tonic-gate { 285*0Sstevel@tonic-gate struct fpu_ctx *fp = &ttolwp(curthread)->lwp_pcb.pcb_fpu; 286*0Sstevel@tonic-gate 287*0Sstevel@tonic-gate ASSERT(curthread->t_preempt >= 1); 288*0Sstevel@tonic-gate ASSERT((fp->fpu_flags & FPU_EN) == 0); 289*0Sstevel@tonic-gate 290*0Sstevel@tonic-gate /* 291*0Sstevel@tonic-gate * Always initialize a new context and initialize the hardware. 292*0Sstevel@tonic-gate */ 293*0Sstevel@tonic-gate installctx(curthread, fp, 294*0Sstevel@tonic-gate fpsave_begin, NULL, fp_new_lwp, fp_new_lwp, NULL, fp_free); 295*0Sstevel@tonic-gate fpinit(); 296*0Sstevel@tonic-gate 297*0Sstevel@tonic-gate /* 298*0Sstevel@tonic-gate * If FPU_VALID is set, it means someone has modified registers via 299*0Sstevel@tonic-gate * /proc. In this case, restore the current lwp's state. 300*0Sstevel@tonic-gate */ 301*0Sstevel@tonic-gate if (fp->fpu_flags & FPU_VALID) 302*0Sstevel@tonic-gate fp_restore(fp); 303*0Sstevel@tonic-gate 304*0Sstevel@tonic-gate ASSERT((fp->fpu_flags & FPU_VALID) == 0); 305*0Sstevel@tonic-gate fp->fpu_flags = FPU_EN; 306*0Sstevel@tonic-gate } 307*0Sstevel@tonic-gate 308*0Sstevel@tonic-gate /* 309*0Sstevel@tonic-gate * This routine is called from trap() when User thread takes No Extension 310*0Sstevel@tonic-gate * Fault. The possiblities are: 311*0Sstevel@tonic-gate * 1. User thread has executed a FP instruction for the first time. 312*0Sstevel@tonic-gate * Save current FPU context if any. Initialize FPU, setup FPU 313*0Sstevel@tonic-gate * context for the thread and enable FP hw. 314*0Sstevel@tonic-gate * 2. Thread's pcb has a valid FPU state: Restore the FPU state and 315*0Sstevel@tonic-gate * enable FP hw. 316*0Sstevel@tonic-gate * 317*0Sstevel@tonic-gate * Note that case #2 is inlined in the trap table. 318*0Sstevel@tonic-gate */ 319*0Sstevel@tonic-gate int 320*0Sstevel@tonic-gate fpnoextflt(struct regs *rp) 321*0Sstevel@tonic-gate { 322*0Sstevel@tonic-gate struct fpu_ctx *fp = &ttolwp(curthread)->lwp_pcb.pcb_fpu; 323*0Sstevel@tonic-gate 324*0Sstevel@tonic-gate #if !defined(__lint) 325*0Sstevel@tonic-gate ASSERT(sizeof (struct fxsave_state) == 512 && 326*0Sstevel@tonic-gate sizeof (struct fnsave_state) == 108); 327*0Sstevel@tonic-gate ASSERT((offsetof(struct fxsave_state, fx_xmm[0]) & 0xf) == 0); 328*0Sstevel@tonic-gate #if defined(__i386) 329*0Sstevel@tonic-gate ASSERT(sizeof (struct fpu) == sizeof (struct __old_fpu)); 330*0Sstevel@tonic-gate #endif /* __i386 */ 331*0Sstevel@tonic-gate #endif /* !__lint */ 332*0Sstevel@tonic-gate 333*0Sstevel@tonic-gate /* 334*0Sstevel@tonic-gate * save area MUST be 16-byte aligned, else will page fault 335*0Sstevel@tonic-gate */ 336*0Sstevel@tonic-gate ASSERT(((uintptr_t)(&fp->fpu_regs.kfpu_u.kfpu_fx) & 0xf) == 0); 337*0Sstevel@tonic-gate 338*0Sstevel@tonic-gate kpreempt_disable(); 339*0Sstevel@tonic-gate /* 340*0Sstevel@tonic-gate * Now we can enable the interrupts. 341*0Sstevel@tonic-gate * (NOTE: fp-no-coprocessor comes thru interrupt gate) 342*0Sstevel@tonic-gate */ 343*0Sstevel@tonic-gate sti(); 344*0Sstevel@tonic-gate 345*0Sstevel@tonic-gate if (!fpu_exists) { /* check for FPU hw exists */ 346*0Sstevel@tonic-gate if (fp_kind == FP_NO) { 347*0Sstevel@tonic-gate uint32_t inst; 348*0Sstevel@tonic-gate 349*0Sstevel@tonic-gate /* 350*0Sstevel@tonic-gate * When the system has no floating point support, 351*0Sstevel@tonic-gate * i.e. no FP hardware and no emulator, skip the 352*0Sstevel@tonic-gate * two kinds of FP instruction that occur in 353*0Sstevel@tonic-gate * fpstart. Allows processes that do no real FP 354*0Sstevel@tonic-gate * to run normally. 355*0Sstevel@tonic-gate */ 356*0Sstevel@tonic-gate if (fuword32((void *)rp->r_pc, &inst) != -1 && 357*0Sstevel@tonic-gate ((inst & 0xFFFF) == 0x7dd9 || 358*0Sstevel@tonic-gate (inst & 0xFFFF) == 0x6dd9)) { 359*0Sstevel@tonic-gate rp->r_pc += 3; 360*0Sstevel@tonic-gate kpreempt_enable(); 361*0Sstevel@tonic-gate return (0); 362*0Sstevel@tonic-gate } 363*0Sstevel@tonic-gate } 364*0Sstevel@tonic-gate 365*0Sstevel@tonic-gate /* 366*0Sstevel@tonic-gate * If we have neither a processor extension nor 367*0Sstevel@tonic-gate * an emulator, kill the process OR panic the kernel. 368*0Sstevel@tonic-gate */ 369*0Sstevel@tonic-gate kpreempt_enable(); 370*0Sstevel@tonic-gate return (1); /* error */ 371*0Sstevel@tonic-gate } 372*0Sstevel@tonic-gate 373*0Sstevel@tonic-gate /* 374*0Sstevel@tonic-gate * A paranoid cross-check: for the SSE case, ensure that %cr4 is 375*0Sstevel@tonic-gate * configured to enable fully fledged (%xmm) fxsave/fxrestor on 376*0Sstevel@tonic-gate * this CPU. For the non-SSE case, ensure that it isn't. 377*0Sstevel@tonic-gate */ 378*0Sstevel@tonic-gate ASSERT((fp_kind == __FP_SSE && (getcr4() & CR4_OSFXSR) == CR4_OSFXSR) || 379*0Sstevel@tonic-gate (fp_kind != __FP_SSE && 380*0Sstevel@tonic-gate (getcr4() & (CR4_OSXMMEXCPT|CR4_OSFXSR)) == 0)); 381*0Sstevel@tonic-gate 382*0Sstevel@tonic-gate if (fp->fpu_flags & FPU_EN) { 383*0Sstevel@tonic-gate /* case 2 */ 384*0Sstevel@tonic-gate fp_restore(fp); 385*0Sstevel@tonic-gate } else { 386*0Sstevel@tonic-gate /* case 1 */ 387*0Sstevel@tonic-gate fp_seed(); 388*0Sstevel@tonic-gate } 389*0Sstevel@tonic-gate kpreempt_enable(); 390*0Sstevel@tonic-gate return (0); 391*0Sstevel@tonic-gate } 392*0Sstevel@tonic-gate 393*0Sstevel@tonic-gate 394*0Sstevel@tonic-gate /* 395*0Sstevel@tonic-gate * Handle a processor extension overrun fault 396*0Sstevel@tonic-gate * Returns non zero for error. 397*0Sstevel@tonic-gate */ 398*0Sstevel@tonic-gate 399*0Sstevel@tonic-gate /* ARGSUSED */ 400*0Sstevel@tonic-gate int 401*0Sstevel@tonic-gate fpextovrflt(struct regs *rp) 402*0Sstevel@tonic-gate { 403*0Sstevel@tonic-gate ulong_t cur_cr0; 404*0Sstevel@tonic-gate 405*0Sstevel@tonic-gate ASSERT(fp_kind != FP_NO); 406*0Sstevel@tonic-gate 407*0Sstevel@tonic-gate cur_cr0 = getcr0(); 408*0Sstevel@tonic-gate fpinit(); /* initialize the FPU hardware */ 409*0Sstevel@tonic-gate setcr0(cur_cr0); 410*0Sstevel@tonic-gate sti(); 411*0Sstevel@tonic-gate 412*0Sstevel@tonic-gate return (1); /* error, send SIGSEGV signal to the thread */ 413*0Sstevel@tonic-gate } 414*0Sstevel@tonic-gate 415*0Sstevel@tonic-gate /* 416*0Sstevel@tonic-gate * Handle a processor extension error fault 417*0Sstevel@tonic-gate * Returns non zero for error. 418*0Sstevel@tonic-gate */ 419*0Sstevel@tonic-gate 420*0Sstevel@tonic-gate /*ARGSUSED*/ 421*0Sstevel@tonic-gate int 422*0Sstevel@tonic-gate fpexterrflt(struct regs *rp) 423*0Sstevel@tonic-gate { 424*0Sstevel@tonic-gate uint32_t fpcwsw; 425*0Sstevel@tonic-gate fpu_ctx_t *fp = &ttolwp(curthread)->lwp_pcb.pcb_fpu; 426*0Sstevel@tonic-gate 427*0Sstevel@tonic-gate ASSERT(fp_kind != FP_NO); 428*0Sstevel@tonic-gate 429*0Sstevel@tonic-gate fpcwsw = fpgetcwsw(); 430*0Sstevel@tonic-gate /* 431*0Sstevel@tonic-gate * Now we can enable the interrupts. 432*0Sstevel@tonic-gate * (NOTE: x87 fp exceptions come thru interrupt gate) 433*0Sstevel@tonic-gate */ 434*0Sstevel@tonic-gate sti(); 435*0Sstevel@tonic-gate 436*0Sstevel@tonic-gate if ((fpcwsw & FPS_ES) == 0) 437*0Sstevel@tonic-gate return (0); /* No exception */ 438*0Sstevel@tonic-gate 439*0Sstevel@tonic-gate if (fpu_exists) { 440*0Sstevel@tonic-gate fp_save(fp); 441*0Sstevel@tonic-gate /* clear exception flags in saved state, as if by fnclex */ 442*0Sstevel@tonic-gate #if defined(__amd64) 443*0Sstevel@tonic-gate fp->fpu_regs.kfpu_u.kfpu_fx.fx_fsw &= ~FPS_SW_EFLAGS; 444*0Sstevel@tonic-gate #else 445*0Sstevel@tonic-gate switch (fp_kind) { 446*0Sstevel@tonic-gate case __FP_SSE: 447*0Sstevel@tonic-gate fp->fpu_regs.kfpu_u.kfpu_fx.fx_fsw &= ~FPS_SW_EFLAGS; 448*0Sstevel@tonic-gate break; 449*0Sstevel@tonic-gate default: 450*0Sstevel@tonic-gate fp->fpu_regs.kfpu_u.kfpu_fn.f_fsw &= ~FPS_SW_EFLAGS; 451*0Sstevel@tonic-gate break; 452*0Sstevel@tonic-gate } 453*0Sstevel@tonic-gate #endif 454*0Sstevel@tonic-gate } 455*0Sstevel@tonic-gate fp->fpu_regs.kfpu_status = fpcwsw & 0xffff; 456*0Sstevel@tonic-gate /* 457*0Sstevel@tonic-gate * "and" the exception flags with the complement of the mask 458*0Sstevel@tonic-gate * bits to determine which exception occurred 459*0Sstevel@tonic-gate */ 460*0Sstevel@tonic-gate return (fpe_sicode(fpcwsw & ~(fpcwsw >> 16) & 0x3f)); 461*0Sstevel@tonic-gate } 462*0Sstevel@tonic-gate 463*0Sstevel@tonic-gate /* 464*0Sstevel@tonic-gate * Handle an SSE/SSE2 precise exception. 465*0Sstevel@tonic-gate * Returns a non-zero sicode for error. 466*0Sstevel@tonic-gate */ 467*0Sstevel@tonic-gate /*ARGSUSED*/ 468*0Sstevel@tonic-gate int 469*0Sstevel@tonic-gate fpsimderrflt(struct regs *rp) 470*0Sstevel@tonic-gate { 471*0Sstevel@tonic-gate uint32_t mxcsr, xmask; 472*0Sstevel@tonic-gate fpu_ctx_t *fp = &ttolwp(curthread)->lwp_pcb.pcb_fpu; 473*0Sstevel@tonic-gate 474*0Sstevel@tonic-gate ASSERT(fp_kind == __FP_SSE); 475*0Sstevel@tonic-gate 476*0Sstevel@tonic-gate mxcsr = fpgetmxcsr(); 477*0Sstevel@tonic-gate if (fpu_exists) { 478*0Sstevel@tonic-gate fp_save(fp); /* save the FPU state */ 479*0Sstevel@tonic-gate fp->fpu_regs.kfpu_status = fp->fpu_regs.kfpu_u.kfpu_fx.fx_fsw; 480*0Sstevel@tonic-gate } else { 481*0Sstevel@tonic-gate fp->fpu_regs.kfpu_status = fpgetcwsw() & 0xffff; 482*0Sstevel@tonic-gate } 483*0Sstevel@tonic-gate fp->fpu_regs.kfpu_xstatus = mxcsr; 484*0Sstevel@tonic-gate 485*0Sstevel@tonic-gate /* 486*0Sstevel@tonic-gate * compute the mask that determines which conditions can cause 487*0Sstevel@tonic-gate * a #xm exception, and use this to clean the status bits so that 488*0Sstevel@tonic-gate * we can identify the true cause of this one. 489*0Sstevel@tonic-gate */ 490*0Sstevel@tonic-gate xmask = (mxcsr >> 7) & SSE_MXCSR_EFLAGS; 491*0Sstevel@tonic-gate return (fpe_simd_sicode((mxcsr & SSE_MXCSR_EFLAGS) & ~xmask)); 492*0Sstevel@tonic-gate } 493*0Sstevel@tonic-gate 494*0Sstevel@tonic-gate /* 495*0Sstevel@tonic-gate * In the unlikely event that someone is relying on this subcode being 496*0Sstevel@tonic-gate * FPE_FLTILL for denormalize exceptions, it can always be patched back 497*0Sstevel@tonic-gate * again to restore old behaviour. 498*0Sstevel@tonic-gate */ 499*0Sstevel@tonic-gate int fpe_fltden = FPE_FLTDEN; 500*0Sstevel@tonic-gate 501*0Sstevel@tonic-gate /* 502*0Sstevel@tonic-gate * Map from the FPU status word to the FP exception si_code. 503*0Sstevel@tonic-gate */ 504*0Sstevel@tonic-gate static int 505*0Sstevel@tonic-gate fpe_sicode(uint_t sw) 506*0Sstevel@tonic-gate { 507*0Sstevel@tonic-gate if (sw & FPS_IE) 508*0Sstevel@tonic-gate return (FPE_FLTINV); 509*0Sstevel@tonic-gate if (sw & FPS_ZE) 510*0Sstevel@tonic-gate return (FPE_FLTDIV); 511*0Sstevel@tonic-gate if (sw & FPS_DE) 512*0Sstevel@tonic-gate return (fpe_fltden); 513*0Sstevel@tonic-gate if (sw & FPS_OE) 514*0Sstevel@tonic-gate return (FPE_FLTOVF); 515*0Sstevel@tonic-gate if (sw & FPS_UE) 516*0Sstevel@tonic-gate return (FPE_FLTUND); 517*0Sstevel@tonic-gate if (sw & FPS_PE) 518*0Sstevel@tonic-gate return (FPE_FLTRES); 519*0Sstevel@tonic-gate return (FPE_FLTINV); /* default si_code for other exceptions */ 520*0Sstevel@tonic-gate } 521*0Sstevel@tonic-gate 522*0Sstevel@tonic-gate /* 523*0Sstevel@tonic-gate * Map from the SSE status word to the FP exception si_code. 524*0Sstevel@tonic-gate */ 525*0Sstevel@tonic-gate static int 526*0Sstevel@tonic-gate fpe_simd_sicode(uint_t sw) 527*0Sstevel@tonic-gate { 528*0Sstevel@tonic-gate if (sw & SSE_IE) 529*0Sstevel@tonic-gate return (FPE_FLTINV); 530*0Sstevel@tonic-gate if (sw & SSE_ZE) 531*0Sstevel@tonic-gate return (FPE_FLTDIV); 532*0Sstevel@tonic-gate if (sw & SSE_DE) 533*0Sstevel@tonic-gate return (FPE_FLTDEN); 534*0Sstevel@tonic-gate if (sw & SSE_OE) 535*0Sstevel@tonic-gate return (FPE_FLTOVF); 536*0Sstevel@tonic-gate if (sw & SSE_UE) 537*0Sstevel@tonic-gate return (FPE_FLTUND); 538*0Sstevel@tonic-gate if (sw & SSE_PE) 539*0Sstevel@tonic-gate return (FPE_FLTRES); 540*0Sstevel@tonic-gate return (FPE_FLTINV); /* default si_code for other exceptions */ 541*0Sstevel@tonic-gate } 542*0Sstevel@tonic-gate 543*0Sstevel@tonic-gate /* 544*0Sstevel@tonic-gate * This routine is invoked as part of libc's __fpstart implementation 545*0Sstevel@tonic-gate * via sysi86(2). 546*0Sstevel@tonic-gate * 547*0Sstevel@tonic-gate * It may be called -before- any context has been assigned in which case 548*0Sstevel@tonic-gate * we try and avoid touching the hardware. Or it may be invoked well 549*0Sstevel@tonic-gate * after the context has been assigned and fiddled with, in which case 550*0Sstevel@tonic-gate * just tweak it directly. 551*0Sstevel@tonic-gate */ 552*0Sstevel@tonic-gate void 553*0Sstevel@tonic-gate fpsetcw(uint16_t fcw, uint32_t mxcsr) 554*0Sstevel@tonic-gate { 555*0Sstevel@tonic-gate struct fpu_ctx *fp = &curthread->t_lwp->lwp_pcb.pcb_fpu; 556*0Sstevel@tonic-gate struct fxsave_state *fx; 557*0Sstevel@tonic-gate 558*0Sstevel@tonic-gate if (!fpu_exists || fp_kind == FP_NO) 559*0Sstevel@tonic-gate return; 560*0Sstevel@tonic-gate 561*0Sstevel@tonic-gate if ((fp->fpu_flags & FPU_EN) == 0) { 562*0Sstevel@tonic-gate if (fcw == FPU_CW_INIT && mxcsr == SSE_MXCSR_INIT) { 563*0Sstevel@tonic-gate /* 564*0Sstevel@tonic-gate * Common case. Floating point unit not yet 565*0Sstevel@tonic-gate * enabled, and kernel already intends to initialize 566*0Sstevel@tonic-gate * the hardware the way the caller wants. 567*0Sstevel@tonic-gate */ 568*0Sstevel@tonic-gate return; 569*0Sstevel@tonic-gate } 570*0Sstevel@tonic-gate /* 571*0Sstevel@tonic-gate * Hmm. Userland wants a different default. 572*0Sstevel@tonic-gate * Do a fake "first trap" to establish the context, then 573*0Sstevel@tonic-gate * handle as if we already had a context before we came in. 574*0Sstevel@tonic-gate */ 575*0Sstevel@tonic-gate kpreempt_disable(); 576*0Sstevel@tonic-gate fp_seed(); 577*0Sstevel@tonic-gate kpreempt_enable(); 578*0Sstevel@tonic-gate } 579*0Sstevel@tonic-gate 580*0Sstevel@tonic-gate /* 581*0Sstevel@tonic-gate * Ensure that the current hardware state is flushed back to the 582*0Sstevel@tonic-gate * pcb, then modify that copy. Next use of the fp will 583*0Sstevel@tonic-gate * restore the context. 584*0Sstevel@tonic-gate */ 585*0Sstevel@tonic-gate fp_save(fp); 586*0Sstevel@tonic-gate 587*0Sstevel@tonic-gate #if defined(__amd64) 588*0Sstevel@tonic-gate fx = &fp->fpu_regs.kfpu_u.kfpu_fx; 589*0Sstevel@tonic-gate fx->fx_fcw = fcw; 590*0Sstevel@tonic-gate fx->fx_mxcsr = sse_mxcsr_mask & mxcsr; 591*0Sstevel@tonic-gate #else 592*0Sstevel@tonic-gate switch (fp_kind) { 593*0Sstevel@tonic-gate case __FP_SSE: 594*0Sstevel@tonic-gate fx = &fp->fpu_regs.kfpu_u.kfpu_fx; 595*0Sstevel@tonic-gate fx->fx_fcw = fcw; 596*0Sstevel@tonic-gate fx->fx_mxcsr = sse_mxcsr_mask & mxcsr; 597*0Sstevel@tonic-gate break; 598*0Sstevel@tonic-gate default: 599*0Sstevel@tonic-gate fp->fpu_regs.kfpu_u.kfpu_fn.f_fcw = fcw; 600*0Sstevel@tonic-gate break; 601*0Sstevel@tonic-gate } 602*0Sstevel@tonic-gate #endif 603*0Sstevel@tonic-gate } 604