1 /* $NetBSD: vfp_init.c,v 1.68 2020/07/13 16:54:03 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2008 ARM Ltd 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of the company may not be used to endorse or promote 16 * products derived from this software without specific prior written 17 * permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY 23 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 25 * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 28 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 29 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include "opt_cputypes.h" 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: vfp_init.c,v 1.68 2020/07/13 16:54:03 riastradh Exp $"); 36 37 #include <sys/param.h> 38 #include <sys/types.h> 39 #include <sys/systm.h> 40 #include <sys/device.h> 41 #include <sys/proc.h> 42 #include <sys/cpu.h> 43 44 #include <arm/locore.h> 45 #include <arm/pcb.h> 46 #include <arm/undefined.h> 47 #include <arm/vfpreg.h> 48 #include <arm/mcontext.h> 49 #include <arm/fpu.h> 50 51 #include <uvm/uvm_extern.h> /* for pmap.h */ 52 53 #include <crypto/aes/aes.h> 54 #include <crypto/aes/arch/arm/aes_neon.h> 55 56 #ifdef FPU_VFP 57 58 #ifdef CPU_CORTEX 59 #define SETFPU __asm(".fpu\tvfpv4") 60 #else 61 #define SETFPU __asm(".fpu\tvfp") 62 #endif 63 SETFPU; 64 65 /* FLDMD <X>, {d0-d15} */ 66 static inline void 67 load_vfpregs_lo(const uint64_t *p) 68 { 69 SETFPU; 70 __asm __volatile(".fpu vfp\n vldmia\t%0, {d0-d15}" :: "r" (p) : "memory"); 71 } 72 73 /* FSTMD <X>, {d0-d15} */ 74 static inline void 75 save_vfpregs_lo(uint64_t *p) 76 { 77 SETFPU; 78 __asm __volatile(".fpu vfp\n vstmia\t%0, {d0-d15}" :: "r" (p) : "memory"); 79 } 80 81 #ifdef CPU_CORTEX 82 /* FLDMD <X>, {d16-d31} */ 83 static inline void 84 load_vfpregs_hi(const uint64_t *p) 85 { 86 SETFPU; 87 __asm __volatile(".fpu neon-vfpv4\n vldmia\t%0, {d16-d31}" :: "r" (&p[16]) : "memory"); 88 } 89 90 /* FLDMD <X>, {d16-d31} */ 91 static inline void 92 save_vfpregs_hi(uint64_t *p) 93 { 94 SETFPU; 95 __asm __volatile(".fpu neon-vfpv4\nvstmia\t%0, {d16-d31}" :: "r" (&p[16]) : "memory"); 96 } 97 #endif 98 99 static inline void 100 load_vfpregs(const struct vfpreg *fregs) 101 { 102 load_vfpregs_lo(fregs->vfp_regs); 103 #ifdef CPU_CORTEX 104 #ifdef CPU_ARM11 105 switch (curcpu()->ci_vfp_id) { 106 case FPU_VFP_CORTEXA5: 107 case FPU_VFP_CORTEXA7: 108 case FPU_VFP_CORTEXA8: 109 case FPU_VFP_CORTEXA9: 110 case FPU_VFP_CORTEXA15: 111 case FPU_VFP_CORTEXA15_QEMU: 112 case FPU_VFP_CORTEXA53: 113 case FPU_VFP_CORTEXA57: 114 #endif 115 load_vfpregs_hi(fregs->vfp_regs); 116 #ifdef CPU_ARM11 117 break; 118 } 119 #endif 120 #endif 121 } 122 123 static inline void 124 save_vfpregs(struct vfpreg *fregs) 125 { 126 save_vfpregs_lo(fregs->vfp_regs); 127 #ifdef CPU_CORTEX 128 #ifdef CPU_ARM11 129 switch (curcpu()->ci_vfp_id) { 130 case FPU_VFP_CORTEXA5: 131 case FPU_VFP_CORTEXA7: 132 case FPU_VFP_CORTEXA8: 133 case FPU_VFP_CORTEXA9: 134 case FPU_VFP_CORTEXA15: 135 case FPU_VFP_CORTEXA15_QEMU: 136 case FPU_VFP_CORTEXA53: 137 case FPU_VFP_CORTEXA57: 138 #endif 139 save_vfpregs_hi(fregs->vfp_regs); 140 #ifdef CPU_ARM11 141 break; 142 } 143 #endif 144 #endif 145 } 146 147 /* The real handler for VFP bounces. */ 148 static int vfp_handler(u_int, u_int, trapframe_t *, int); 149 #ifdef CPU_CORTEX 150 static int neon_handler(u_int, u_int, trapframe_t *, int); 151 #endif 152 153 static void vfp_state_load(lwp_t *, u_int); 154 static void vfp_state_save(lwp_t *); 155 static void vfp_state_release(lwp_t *); 156 157 const pcu_ops_t arm_vfp_ops = { 158 .pcu_id = PCU_FPU, 159 .pcu_state_save = vfp_state_save, 160 .pcu_state_load = vfp_state_load, 161 .pcu_state_release = vfp_state_release, 162 }; 163 164 /* determine what bits can be changed */ 165 uint32_t vfp_fpscr_changable = VFP_FPSCR_CSUM; 166 /* default to run fast */ 167 uint32_t vfp_fpscr_default = (VFP_FPSCR_DN | VFP_FPSCR_FZ | VFP_FPSCR_RN); 168 169 /* 170 * Used to test for a VFP. The following function is installed as a coproc10 171 * handler on the undefined instruction vector and then we issue a VFP 172 * instruction. If undefined_test is non zero then the VFP did not handle 173 * the instruction so must be absent, or disabled. 174 */ 175 176 static int undefined_test; 177 178 static int 179 vfp_test(u_int address, u_int insn, trapframe_t *frame, int fault_code) 180 { 181 182 frame->tf_pc += INSN_SIZE; 183 ++undefined_test; 184 return 0; 185 } 186 187 #else 188 /* determine what bits can be changed */ 189 uint32_t vfp_fpscr_changable = VFP_FPSCR_CSUM|VFP_FPSCR_ESUM|VFP_FPSCR_RMODE; 190 #endif /* FPU_VFP */ 191 192 static int 193 vfp_fpscr_handler(u_int address, u_int insn, trapframe_t *frame, int fault_code) 194 { 195 struct lwp * const l = curlwp; 196 const u_int regno = (insn >> 12) & 0xf; 197 /* 198 * Only match move to/from the FPSCR register and we 199 * can't be using the SP,LR,PC as a source. 200 */ 201 if ((insn & 0xffef0fff) != 0xeee10a10 || regno > 12) 202 return 1; 203 204 struct pcb * const pcb = lwp_getpcb(l); 205 206 #ifdef FPU_VFP 207 /* 208 * If FPU is valid somewhere, let's just reenable VFP and 209 * retry the instruction (only safe thing to do since the 210 * pcb has a stale copy). 211 */ 212 if (pcb->pcb_vfp.vfp_fpexc & VFP_FPEXC_EN) 213 return 1; 214 215 if (__predict_false(!vfp_used_p(l))) { 216 pcb->pcb_vfp.vfp_fpscr = vfp_fpscr_default; 217 } 218 #endif 219 220 /* 221 * We now know the pcb has the saved copy. 222 */ 223 register_t * const regp = &frame->tf_r0 + regno; 224 if (insn & 0x00100000) { 225 *regp = pcb->pcb_vfp.vfp_fpscr; 226 } else { 227 pcb->pcb_vfp.vfp_fpscr &= ~vfp_fpscr_changable; 228 pcb->pcb_vfp.vfp_fpscr |= *regp & vfp_fpscr_changable; 229 } 230 231 curcpu()->ci_vfp_evs[0].ev_count++; 232 233 frame->tf_pc += INSN_SIZE; 234 return 0; 235 } 236 237 #ifndef FPU_VFP 238 /* 239 * If we don't want VFP support, we still need to handle emulating VFP FPSCR 240 * instructions. 241 */ 242 void 243 vfp_attach(struct cpu_info *ci) 244 { 245 if (CPU_IS_PRIMARY(ci)) { 246 install_coproc_handler(VFP_COPROC, vfp_fpscr_handler); 247 } 248 evcnt_attach_dynamic(&ci->ci_vfp_evs[0], EVCNT_TYPE_TRAP, NULL, 249 ci->ci_cpuname, "vfp fpscr traps"); 250 } 251 252 #else 253 void 254 vfp_attach(struct cpu_info *ci) 255 { 256 const char *model = NULL; 257 258 if (CPU_ID_ARM11_P(ci->ci_arm_cpuid) 259 || CPU_ID_MV88SV58XX_P(ci->ci_arm_cpuid) 260 || CPU_ID_CORTEX_P(ci->ci_arm_cpuid)) { 261 #if 0 262 const uint32_t nsacr = armreg_nsacr_read(); 263 const uint32_t nsacr_vfp = __BITS(VFP_COPROC,VFP_COPROC2); 264 if ((nsacr & nsacr_vfp) != nsacr_vfp) { 265 aprint_normal_dev(ci->ci_dev, 266 "VFP access denied (NSACR=%#x)\n", nsacr); 267 if (CPU_IS_PRIMARY(ci)) 268 install_coproc_handler(VFP_COPROC, vfp_fpscr_handler); 269 ci->ci_vfp_id = 0; 270 evcnt_attach_dynamic(&ci->ci_vfp_evs[0], 271 EVCNT_TYPE_TRAP, NULL, ci->ci_cpuname, 272 "vfp fpscr traps"); 273 return; 274 } 275 #endif 276 const uint32_t cpacr_vfp = CPACR_CPn(VFP_COPROC); 277 const uint32_t cpacr_vfp2 = CPACR_CPn(VFP_COPROC2); 278 279 /* 280 * We first need to enable access to the coprocessors. 281 */ 282 uint32_t cpacr = armreg_cpacr_read(); 283 cpacr |= __SHIFTIN(CPACR_ALL, cpacr_vfp); 284 cpacr |= __SHIFTIN(CPACR_ALL, cpacr_vfp2); 285 armreg_cpacr_write(cpacr); 286 287 arm_isb(); 288 289 /* 290 * If we could enable them, then they exist. 291 */ 292 cpacr = armreg_cpacr_read(); 293 bool vfp_p = __SHIFTOUT(cpacr, cpacr_vfp2) == CPACR_ALL 294 && __SHIFTOUT(cpacr, cpacr_vfp) == CPACR_ALL; 295 if (!vfp_p) { 296 aprint_normal_dev(ci->ci_dev, 297 "VFP access denied (CPACR=%#x)\n", cpacr); 298 if (CPU_IS_PRIMARY(ci)) 299 install_coproc_handler(VFP_COPROC, vfp_fpscr_handler); 300 ci->ci_vfp_id = 0; 301 evcnt_attach_dynamic(&ci->ci_vfp_evs[0], 302 EVCNT_TYPE_TRAP, NULL, ci->ci_cpuname, 303 "vfp fpscr traps"); 304 return; 305 } 306 } 307 308 void *uh = install_coproc_handler(VFP_COPROC, vfp_test); 309 310 undefined_test = 0; 311 312 const uint32_t fpsid = armreg_fpsid_read(); 313 314 remove_coproc_handler(uh); 315 316 if (undefined_test != 0) { 317 aprint_normal_dev(ci->ci_dev, "No VFP detected\n"); 318 if (CPU_IS_PRIMARY(ci)) 319 install_coproc_handler(VFP_COPROC, vfp_fpscr_handler); 320 ci->ci_vfp_id = 0; 321 return; 322 } 323 324 ci->ci_vfp_id = fpsid; 325 switch (fpsid & ~ VFP_FPSID_REV_MSK) { 326 case FPU_VFP10_ARM10E: 327 model = "VFP10 R1"; 328 break; 329 case FPU_VFP11_ARM11: 330 model = "VFP11"; 331 break; 332 case FPU_VFP_MV88SV58XX: 333 model = "VFP3"; 334 break; 335 case FPU_VFP_CORTEXA5: 336 case FPU_VFP_CORTEXA7: 337 case FPU_VFP_CORTEXA8: 338 case FPU_VFP_CORTEXA9: 339 case FPU_VFP_CORTEXA12: 340 case FPU_VFP_CORTEXA15: 341 case FPU_VFP_CORTEXA15_QEMU: 342 case FPU_VFP_CORTEXA17: 343 case FPU_VFP_CORTEXA53: 344 case FPU_VFP_CORTEXA57: 345 if (armreg_cpacr_read() & CPACR_V7_ASEDIS) { 346 model = "VFP 4.0+"; 347 } else { 348 model = "NEON MPE (VFP 3.0+)"; 349 cpu_neon_present = 1; 350 } 351 break; 352 default: 353 aprint_normal_dev(ci->ci_dev, "unrecognized VFP version %#x\n", 354 fpsid); 355 if (CPU_IS_PRIMARY(ci)) 356 install_coproc_handler(VFP_COPROC, vfp_fpscr_handler); 357 vfp_fpscr_changable = VFP_FPSCR_CSUM|VFP_FPSCR_ESUM 358 |VFP_FPSCR_RMODE; 359 vfp_fpscr_default = 0; 360 return; 361 } 362 363 cpu_fpu_present = 1; 364 cpu_media_and_vfp_features[0] = armreg_mvfr0_read(); 365 cpu_media_and_vfp_features[1] = armreg_mvfr1_read(); 366 if (fpsid != 0) { 367 uint32_t f0 = armreg_mvfr0_read(); 368 uint32_t f1 = armreg_mvfr1_read(); 369 aprint_normal("vfp%d at %s: %s%s%s%s%s\n", 370 device_unit(ci->ci_dev), 371 device_xname(ci->ci_dev), 372 model, 373 ((f0 & ARM_MVFR0_ROUNDING_MASK) ? ", rounding" : ""), 374 ((f0 & ARM_MVFR0_EXCEPT_MASK) ? ", exceptions" : ""), 375 ((f1 & ARM_MVFR1_D_NAN_MASK) ? ", NaN propagation" : ""), 376 ((f1 & ARM_MVFR1_FTZ_MASK) ? ", denormals" : "")); 377 aprint_debug("vfp%d: mvfr: [0]=%#x [1]=%#x\n", 378 device_unit(ci->ci_dev), f0, f1); 379 if (CPU_IS_PRIMARY(ci)) { 380 if (f0 & ARM_MVFR0_ROUNDING_MASK) { 381 vfp_fpscr_changable |= VFP_FPSCR_RMODE; 382 } 383 if (f1 & ARM_MVFR0_EXCEPT_MASK) { 384 vfp_fpscr_changable |= VFP_FPSCR_ESUM; 385 } 386 // If hardware supports propagation of NaNs, select it. 387 if (f1 & ARM_MVFR1_D_NAN_MASK) { 388 vfp_fpscr_default &= ~VFP_FPSCR_DN; 389 vfp_fpscr_changable |= VFP_FPSCR_DN; 390 } 391 // If hardware supports denormalized numbers, use it. 392 if (cpu_media_and_vfp_features[1] & ARM_MVFR1_FTZ_MASK) { 393 vfp_fpscr_default &= ~VFP_FPSCR_FZ; 394 vfp_fpscr_changable |= VFP_FPSCR_FZ; 395 } 396 } 397 } 398 evcnt_attach_dynamic(&ci->ci_vfp_evs[0], EVCNT_TYPE_MISC, NULL, 399 ci->ci_cpuname, "vfp coproc use"); 400 evcnt_attach_dynamic(&ci->ci_vfp_evs[1], EVCNT_TYPE_MISC, NULL, 401 ci->ci_cpuname, "vfp coproc re-use"); 402 evcnt_attach_dynamic(&ci->ci_vfp_evs[2], EVCNT_TYPE_TRAP, NULL, 403 ci->ci_cpuname, "vfp coproc fault"); 404 if (CPU_IS_PRIMARY(ci)) { 405 install_coproc_handler(VFP_COPROC, vfp_handler); 406 install_coproc_handler(VFP_COPROC2, vfp_handler); 407 #ifdef CPU_CORTEX 408 if (cpu_neon_present) { 409 install_coproc_handler(CORE_UNKNOWN_HANDLER, 410 neon_handler); 411 aes_md_init(&aes_neon_impl); 412 } 413 #endif 414 } 415 } 416 417 /* The real handler for VFP bounces. */ 418 static int 419 vfp_handler(u_int address, u_int insn, trapframe_t *frame, int fault_code) 420 { 421 struct cpu_info * const ci = curcpu(); 422 423 /* This shouldn't ever happen. */ 424 if (fault_code != FAULT_USER) 425 panic("VFP fault at %#x in non-user mode", frame->tf_pc); 426 427 if (ci->ci_vfp_id == 0) { 428 /* No VFP detected, just fault. */ 429 return 1; 430 } 431 432 /* 433 * If we already own the FPU and it's enabled (and no exception), raise 434 * SIGILL. If there is an exception, drop through to raise a SIGFPE. 435 */ 436 if (curcpu()->ci_pcu_curlwp[PCU_FPU] == curlwp 437 && (armreg_fpexc_read() & (VFP_FPEXC_EX|VFP_FPEXC_EN)) == VFP_FPEXC_EN) 438 return 1; 439 440 /* 441 * Make sure we own the FP. 442 */ 443 pcu_load(&arm_vfp_ops); 444 445 uint32_t fpexc = armreg_fpexc_read(); 446 if (fpexc & VFP_FPEXC_EX) { 447 ksiginfo_t ksi; 448 KASSERT(fpexc & VFP_FPEXC_EN); 449 450 curcpu()->ci_vfp_evs[2].ev_count++; 451 452 /* 453 * Need the clear the exception condition so any signal 454 * and future use can proceed. 455 */ 456 armreg_fpexc_write(fpexc & ~(VFP_FPEXC_EX|VFP_FPEXC_FSUM)); 457 458 pcu_save(&arm_vfp_ops, curlwp); 459 460 /* 461 * XXX Need to emulate bounce instructions here to get correct 462 * XXX exception codes, etc. 463 */ 464 KSI_INIT_TRAP(&ksi); 465 ksi.ksi_signo = SIGFPE; 466 if (fpexc & VFP_FPEXC_IXF) 467 ksi.ksi_code = FPE_FLTRES; 468 else if (fpexc & VFP_FPEXC_UFF) 469 ksi.ksi_code = FPE_FLTUND; 470 else if (fpexc & VFP_FPEXC_OFF) 471 ksi.ksi_code = FPE_FLTOVF; 472 else if (fpexc & VFP_FPEXC_DZF) 473 ksi.ksi_code = FPE_FLTDIV; 474 else if (fpexc & VFP_FPEXC_IOF) 475 ksi.ksi_code = FPE_FLTINV; 476 ksi.ksi_addr = (uint32_t *)address; 477 ksi.ksi_trap = 0; 478 trapsignal(curlwp, &ksi); 479 return 0; 480 } 481 482 /* Need to restart the faulted instruction. */ 483 // frame->tf_pc -= INSN_SIZE; 484 return 0; 485 } 486 487 #ifdef CPU_CORTEX 488 /* The real handler for NEON bounces. */ 489 static int 490 neon_handler(u_int address, u_int insn, trapframe_t *frame, int fault_code) 491 { 492 struct cpu_info * const ci = curcpu(); 493 494 if (ci->ci_vfp_id == 0) 495 /* No VFP detected, just fault. */ 496 return 1; 497 498 if ((insn & 0xfe000000) != 0xf2000000 499 && (insn & 0xfe000000) != 0xf4000000) 500 /* Not NEON instruction, just fault. */ 501 return 1; 502 503 /* This shouldn't ever happen. */ 504 if (fault_code != FAULT_USER) 505 panic("NEON fault in non-user mode"); 506 507 /* if we already own the FPU and it's enabled, raise SIGILL */ 508 if (curcpu()->ci_pcu_curlwp[PCU_FPU] == curlwp 509 && (armreg_fpexc_read() & VFP_FPEXC_EN) != 0) 510 return 1; 511 512 pcu_load(&arm_vfp_ops); 513 514 /* Need to restart the faulted instruction. */ 515 // frame->tf_pc -= INSN_SIZE; 516 return 0; 517 } 518 #endif 519 520 static void 521 vfp_state_load(lwp_t *l, u_int flags) 522 { 523 struct pcb * const pcb = lwp_getpcb(l); 524 struct vfpreg * const fregs = &pcb->pcb_vfp; 525 526 /* 527 * Instrument VFP usage -- if a process has not previously 528 * used the VFP, mark it as having used VFP for the first time, 529 * and count this event. 530 * 531 * If a process has used the VFP, count a "used VFP, and took 532 * a trap to use it again" event. 533 */ 534 if (__predict_false((flags & PCU_VALID) == 0)) { 535 curcpu()->ci_vfp_evs[0].ev_count++; 536 pcb->pcb_vfp.vfp_fpscr = vfp_fpscr_default; 537 } else { 538 curcpu()->ci_vfp_evs[1].ev_count++; 539 } 540 541 KASSERT((armreg_fpexc_read() & VFP_FPEXC_EN) == 0); 542 /* 543 * If the VFP is already enabled we must be bouncing an instruction. 544 */ 545 if (flags & PCU_REENABLE) { 546 uint32_t fpexc = armreg_fpexc_read(); 547 armreg_fpexc_write(fpexc | VFP_FPEXC_EN); 548 fregs->vfp_fpexc |= VFP_FPEXC_EN; 549 return; 550 } 551 KASSERT((fregs->vfp_fpexc & VFP_FPEXC_EN) == 0); 552 553 /* 554 * Load and Enable the VFP (so that we can write the registers). 555 */ 556 fregs->vfp_fpexc |= VFP_FPEXC_EN; 557 armreg_fpexc_write(fregs->vfp_fpexc); 558 KASSERT(curcpu()->ci_pcu_curlwp[PCU_FPU] == NULL); 559 KASSERT(l->l_pcu_cpu[PCU_FPU] == NULL); 560 561 load_vfpregs(fregs); 562 armreg_fpscr_write(fregs->vfp_fpscr); 563 564 if (fregs->vfp_fpexc & VFP_FPEXC_EX) { 565 /* Need to restore the exception handling state. */ 566 armreg_fpinst_write(fregs->vfp_fpinst); 567 if (fregs->vfp_fpexc & VFP_FPEXC_FP2V) 568 armreg_fpinst2_write(fregs->vfp_fpinst2); 569 } 570 } 571 572 void 573 vfp_state_save(lwp_t *l) 574 { 575 struct pcb * const pcb = lwp_getpcb(l); 576 struct vfpreg * const fregs = &pcb->pcb_vfp; 577 uint32_t fpexc = armreg_fpexc_read(); 578 579 KASSERT(curcpu()->ci_pcu_curlwp[PCU_FPU] == l); 580 KASSERT(curcpu() == l->l_pcu_cpu[PCU_FPU]); 581 KASSERT(curlwp == l || curlwp->l_pcu_cpu[PCU_FPU] != curcpu()); 582 /* 583 * Enable the VFP (so we can read the registers). 584 * Make sure the exception bit is cleared so that we can 585 * safely dump the registers. 586 */ 587 armreg_fpexc_write((fpexc | VFP_FPEXC_EN) & ~VFP_FPEXC_EX); 588 589 fregs->vfp_fpexc = fpexc; 590 if (fpexc & VFP_FPEXC_EX) { 591 /* Need to save the exception handling state */ 592 fregs->vfp_fpinst = armreg_fpinst_read(); 593 if (fpexc & VFP_FPEXC_FP2V) 594 fregs->vfp_fpinst2 = armreg_fpinst2_read(); 595 } 596 fregs->vfp_fpscr = armreg_fpscr_read(); 597 save_vfpregs(fregs); 598 599 /* Disable the VFP. */ 600 armreg_fpexc_write(fpexc & ~VFP_FPEXC_EN); 601 } 602 603 void 604 vfp_state_release(lwp_t *l) 605 { 606 struct pcb * const pcb = lwp_getpcb(l); 607 608 /* 609 * Now mark the VFP as disabled (and our state 610 * has been already saved or is being discarded). 611 */ 612 pcb->pcb_vfp.vfp_fpexc &= ~VFP_FPEXC_EN; 613 614 /* 615 * Turn off the FPU so the next time a VFP instruction is issued 616 * an exception happens. We don't know if this LWP's state was 617 * loaded but if we turned off the FPU for some other LWP, when 618 * pcu_load invokes vfp_state_load it will see that VFP_FPEXC_EN 619 * is still set so it just restore fpexc and return since its 620 * contents are still sitting in the VFP. 621 */ 622 armreg_fpexc_write(armreg_fpexc_read() & ~VFP_FPEXC_EN); 623 } 624 625 void 626 vfp_savecontext(lwp_t *l) 627 { 628 pcu_save(&arm_vfp_ops, l); 629 } 630 631 void 632 vfp_discardcontext(lwp_t *l, bool used_p) 633 { 634 pcu_discard(&arm_vfp_ops, l, used_p); 635 } 636 637 bool 638 vfp_used_p(const lwp_t *l) 639 { 640 return pcu_valid_p(&arm_vfp_ops, l); 641 } 642 643 void 644 vfp_getcontext(struct lwp *l, mcontext_t *mcp, int *flagsp) 645 { 646 if (vfp_used_p(l)) { 647 const struct pcb * const pcb = lwp_getpcb(l); 648 649 pcu_save(&arm_vfp_ops, l); 650 mcp->__fpu.__vfpregs.__vfp_fpscr = pcb->pcb_vfp.vfp_fpscr; 651 memcpy(mcp->__fpu.__vfpregs.__vfp_fstmx, pcb->pcb_vfp.vfp_regs, 652 sizeof(mcp->__fpu.__vfpregs.__vfp_fstmx)); 653 *flagsp |= _UC_FPU|_UC_ARM_VFP; 654 } 655 } 656 657 void 658 vfp_setcontext(struct lwp *l, const mcontext_t *mcp) 659 { 660 struct pcb * const pcb = lwp_getpcb(l); 661 662 pcu_discard(&arm_vfp_ops, l, true); 663 pcb->pcb_vfp.vfp_fpscr = mcp->__fpu.__vfpregs.__vfp_fpscr; 664 memcpy(pcb->pcb_vfp.vfp_regs, mcp->__fpu.__vfpregs.__vfp_fstmx, 665 sizeof(mcp->__fpu.__vfpregs.__vfp_fstmx)); 666 } 667 668 void 669 fpu_kern_enter(void) 670 { 671 struct cpu_info *ci; 672 uint32_t fpexc; 673 int s; 674 675 /* 676 * Block interrupts up to IPL_VM. We must block preemption 677 * since -- if this is a user thread -- there is nowhere to 678 * save the kernel fpu state, and if we want this to be usable 679 * in interrupts, we can't let interrupts interfere with the 680 * fpu state in use since there's nowhere for them to save it. 681 */ 682 s = splvm(); 683 ci = curcpu(); 684 KASSERTMSG(ci->ci_cpl <= IPL_VM, "cpl=%d", ci->ci_cpl); 685 KASSERT(ci->ci_kfpu_spl == -1); 686 ci->ci_kfpu_spl = s; 687 688 /* Save any fpu state on the current CPU. */ 689 pcu_save_all_on_cpu(); 690 691 /* Enable the fpu. */ 692 fpexc = armreg_fpexc_read(); 693 fpexc |= VFP_FPEXC_EN; 694 fpexc &= ~VFP_FPEXC_EX; 695 armreg_fpexc_write(fpexc); 696 } 697 698 void 699 fpu_kern_leave(void) 700 { 701 static const struct vfpreg zero_vfpreg; 702 struct cpu_info *ci = curcpu(); 703 int s; 704 uint32_t fpexc; 705 706 KASSERT(ci->ci_cpl == IPL_VM); 707 KASSERT(ci->ci_kfpu_spl != -1); 708 709 /* 710 * Zero the fpu registers; otherwise we might leak secrets 711 * through Spectre-class attacks to userland, even if there are 712 * no bugs in fpu state management. 713 */ 714 load_vfpregs(&zero_vfpreg); 715 716 /* 717 * Disable the fpu so that the kernel can't accidentally use 718 * it again. 719 */ 720 fpexc = armreg_fpexc_read(); 721 fpexc &= ~VFP_FPEXC_EN; 722 armreg_fpexc_write(fpexc); 723 724 /* Restore interrupts. */ 725 s = ci->ci_kfpu_spl; 726 ci->ci_kfpu_spl = -1; 727 splx(s); 728 } 729 730 #endif /* FPU_VFP */ 731