1 /* $NetBSD: vfp_init.c,v 1.74 2021/06/01 00:30:22 rin Exp $ */ 2 3 /* 4 * Copyright (c) 2008 ARM Ltd 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of the company may not be used to endorse or promote 16 * products derived from this software without specific prior written 17 * permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY 23 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 25 * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 28 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 29 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include "opt_cputypes.h" 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: vfp_init.c,v 1.74 2021/06/01 00:30:22 rin Exp $"); 36 37 #include <sys/param.h> 38 #include <sys/types.h> 39 #include <sys/systm.h> 40 #include <sys/device.h> 41 #include <sys/kthread.h> 42 #include <sys/proc.h> 43 #include <sys/cpu.h> 44 45 #include <arm/locore.h> 46 #include <arm/pcb.h> 47 #include <arm/undefined.h> 48 #include <arm/vfpreg.h> 49 #include <arm/mcontext.h> 50 #include <arm/fpu.h> 51 52 #include <uvm/uvm_extern.h> /* for pmap.h */ 53 54 #include <crypto/aes/aes_impl.h> 55 #include <crypto/aes/arch/arm/aes_neon.h> 56 #include <crypto/chacha/arch/arm/chacha_neon.h> 57 #include <crypto/chacha/chacha_impl.h> 58 59 #ifdef FPU_VFP 60 61 #ifdef CPU_CORTEX 62 #define SETFPU __asm(".fpu\tvfpv4") 63 #else 64 #define SETFPU __asm(".fpu\tvfp") 65 #endif 66 SETFPU; 67 68 /* FLDMD <X>, {d0-d15} */ 69 static inline void 70 load_vfpregs_lo(const uint64_t *p) 71 { 72 SETFPU; 73 __asm __volatile(".fpu vfp\n vldmia\t%0, {d0-d15}" :: "r" (p) : "memory"); 74 } 75 76 /* FSTMD <X>, {d0-d15} */ 77 static inline void 78 save_vfpregs_lo(uint64_t *p) 79 { 80 SETFPU; 81 __asm __volatile(".fpu vfp\n vstmia\t%0, {d0-d15}" :: "r" (p) : "memory"); 82 } 83 84 #ifdef CPU_CORTEX 85 /* FLDMD <X>, {d16-d31} */ 86 static inline void 87 load_vfpregs_hi(const uint64_t *p) 88 { 89 SETFPU; 90 __asm __volatile(".fpu neon-vfpv4\n vldmia\t%0, {d16-d31}" :: "r" (&p[16]) : "memory"); 91 } 92 93 /* FLDMD <X>, {d16-d31} */ 94 static inline void 95 save_vfpregs_hi(uint64_t *p) 96 { 97 SETFPU; 98 __asm __volatile(".fpu neon-vfpv4\nvstmia\t%0, {d16-d31}" :: "r" (&p[16]) : "memory"); 99 } 100 #endif 101 102 static inline void 103 load_vfpregs(const struct vfpreg *fregs) 104 { 105 load_vfpregs_lo(fregs->vfp_regs); 106 #ifdef CPU_CORTEX 107 #ifdef CPU_ARM11 108 switch (curcpu()->ci_vfp_id) { 109 case FPU_VFP_CORTEXA5: 110 case FPU_VFP_CORTEXA7: 111 case FPU_VFP_CORTEXA8: 112 case FPU_VFP_CORTEXA9: 113 case FPU_VFP_CORTEXA15: 114 case FPU_VFP_CORTEXA15_QEMU: 115 case FPU_VFP_CORTEXA53: 116 case FPU_VFP_CORTEXA57: 117 #endif 118 load_vfpregs_hi(fregs->vfp_regs); 119 #ifdef CPU_ARM11 120 break; 121 } 122 #endif 123 #endif 124 } 125 126 static inline void 127 save_vfpregs(struct vfpreg *fregs) 128 { 129 save_vfpregs_lo(fregs->vfp_regs); 130 #ifdef CPU_CORTEX 131 #ifdef CPU_ARM11 132 switch (curcpu()->ci_vfp_id) { 133 case FPU_VFP_CORTEXA5: 134 case FPU_VFP_CORTEXA7: 135 case FPU_VFP_CORTEXA8: 136 case FPU_VFP_CORTEXA9: 137 case FPU_VFP_CORTEXA15: 138 case FPU_VFP_CORTEXA15_QEMU: 139 case FPU_VFP_CORTEXA53: 140 case FPU_VFP_CORTEXA57: 141 #endif 142 save_vfpregs_hi(fregs->vfp_regs); 143 #ifdef CPU_ARM11 144 break; 145 } 146 #endif 147 #endif 148 } 149 150 /* The real handler for VFP bounces. */ 151 static int vfp_handler(u_int, u_int, trapframe_t *, int); 152 #ifdef CPU_CORTEX 153 static int neon_handler(u_int, u_int, trapframe_t *, int); 154 #endif 155 156 static void vfp_state_load(lwp_t *, u_int); 157 static void vfp_state_save(lwp_t *); 158 static void vfp_state_release(lwp_t *); 159 160 const pcu_ops_t arm_vfp_ops = { 161 .pcu_id = PCU_FPU, 162 .pcu_state_save = vfp_state_save, 163 .pcu_state_load = vfp_state_load, 164 .pcu_state_release = vfp_state_release, 165 }; 166 167 /* determine what bits can be changed */ 168 uint32_t vfp_fpscr_changable = VFP_FPSCR_CSUM; 169 /* default to run fast */ 170 uint32_t vfp_fpscr_default = (VFP_FPSCR_DN | VFP_FPSCR_FZ | VFP_FPSCR_RN); 171 172 /* 173 * Used to test for a VFP. The following function is installed as a coproc10 174 * handler on the undefined instruction vector and then we issue a VFP 175 * instruction. If undefined_test is non zero then the VFP did not handle 176 * the instruction so must be absent, or disabled. 177 */ 178 179 static int undefined_test; 180 181 static int 182 vfp_test(u_int address, u_int insn, trapframe_t *frame, int fault_code) 183 { 184 185 frame->tf_pc += INSN_SIZE; 186 ++undefined_test; 187 return 0; 188 } 189 190 #else 191 /* determine what bits can be changed */ 192 uint32_t vfp_fpscr_changable = VFP_FPSCR_CSUM|VFP_FPSCR_ESUM|VFP_FPSCR_RMODE; 193 #endif /* FPU_VFP */ 194 195 static int 196 vfp_fpscr_handler(u_int address, u_int insn, trapframe_t *frame, int fault_code) 197 { 198 struct lwp * const l = curlwp; 199 const u_int regno = (insn >> 12) & 0xf; 200 /* 201 * Only match move to/from the FPSCR register and we 202 * can't be using the SP,LR,PC as a source. 203 */ 204 if ((insn & 0xffef0fff) != 0xeee10a10 || regno > 12) 205 return 1; 206 207 struct pcb * const pcb = lwp_getpcb(l); 208 209 #ifdef FPU_VFP 210 /* 211 * If FPU is valid somewhere, let's just reenable VFP and 212 * retry the instruction (only safe thing to do since the 213 * pcb has a stale copy). 214 */ 215 if (pcb->pcb_vfp.vfp_fpexc & VFP_FPEXC_EN) 216 return 1; 217 218 if (__predict_false(!vfp_used_p(l))) { 219 pcb->pcb_vfp.vfp_fpscr = vfp_fpscr_default; 220 } 221 #endif 222 223 /* 224 * We now know the pcb has the saved copy. 225 */ 226 register_t * const regp = &frame->tf_r0 + regno; 227 if (insn & 0x00100000) { 228 *regp = pcb->pcb_vfp.vfp_fpscr; 229 } else { 230 pcb->pcb_vfp.vfp_fpscr &= ~vfp_fpscr_changable; 231 pcb->pcb_vfp.vfp_fpscr |= *regp & vfp_fpscr_changable; 232 } 233 234 curcpu()->ci_vfp_evs[0].ev_count++; 235 236 frame->tf_pc += INSN_SIZE; 237 return 0; 238 } 239 240 #ifndef FPU_VFP 241 /* 242 * If we don't want VFP support, we still need to handle emulating VFP FPSCR 243 * instructions. 244 */ 245 void 246 vfp_attach(struct cpu_info *ci) 247 { 248 if (CPU_IS_PRIMARY(ci)) { 249 install_coproc_handler(VFP_COPROC, vfp_fpscr_handler); 250 } 251 evcnt_attach_dynamic(&ci->ci_vfp_evs[0], EVCNT_TYPE_TRAP, NULL, 252 ci->ci_cpuname, "vfp fpscr traps"); 253 } 254 255 #else 256 void 257 vfp_attach(struct cpu_info *ci) 258 { 259 const char *model = NULL; 260 261 if (CPU_ID_ARM11_P(ci->ci_arm_cpuid) 262 || CPU_ID_MV88SV58XX_P(ci->ci_arm_cpuid) 263 || CPU_ID_CORTEX_P(ci->ci_arm_cpuid)) { 264 #if 0 265 const uint32_t nsacr = armreg_nsacr_read(); 266 const uint32_t nsacr_vfp = __BITS(VFP_COPROC,VFP_COPROC2); 267 if ((nsacr & nsacr_vfp) != nsacr_vfp) { 268 aprint_normal_dev(ci->ci_dev, 269 "VFP access denied (NSACR=%#x)\n", nsacr); 270 if (CPU_IS_PRIMARY(ci)) 271 install_coproc_handler(VFP_COPROC, vfp_fpscr_handler); 272 ci->ci_vfp_id = 0; 273 evcnt_attach_dynamic(&ci->ci_vfp_evs[0], 274 EVCNT_TYPE_TRAP, NULL, ci->ci_cpuname, 275 "vfp fpscr traps"); 276 return; 277 } 278 #endif 279 const uint32_t cpacr_vfp = CPACR_CPn(VFP_COPROC); 280 const uint32_t cpacr_vfp2 = CPACR_CPn(VFP_COPROC2); 281 282 /* 283 * We first need to enable access to the coprocessors. 284 */ 285 uint32_t cpacr = armreg_cpacr_read(); 286 cpacr |= __SHIFTIN(CPACR_ALL, cpacr_vfp); 287 cpacr |= __SHIFTIN(CPACR_ALL, cpacr_vfp2); 288 armreg_cpacr_write(cpacr); 289 290 isb(); 291 292 /* 293 * If we could enable them, then they exist. 294 */ 295 cpacr = armreg_cpacr_read(); 296 bool vfp_p = __SHIFTOUT(cpacr, cpacr_vfp2) == CPACR_ALL 297 && __SHIFTOUT(cpacr, cpacr_vfp) == CPACR_ALL; 298 if (!vfp_p) { 299 aprint_normal_dev(ci->ci_dev, 300 "VFP access denied (CPACR=%#x)\n", cpacr); 301 if (CPU_IS_PRIMARY(ci)) 302 install_coproc_handler(VFP_COPROC, vfp_fpscr_handler); 303 ci->ci_vfp_id = 0; 304 evcnt_attach_dynamic(&ci->ci_vfp_evs[0], 305 EVCNT_TYPE_TRAP, NULL, ci->ci_cpuname, 306 "vfp fpscr traps"); 307 return; 308 } 309 } 310 311 void *uh = install_coproc_handler(VFP_COPROC, vfp_test); 312 313 undefined_test = 0; 314 315 const uint32_t fpsid = armreg_fpsid_read(); 316 317 remove_coproc_handler(uh); 318 319 if (undefined_test != 0) { 320 aprint_normal_dev(ci->ci_dev, "No VFP detected\n"); 321 if (CPU_IS_PRIMARY(ci)) 322 install_coproc_handler(VFP_COPROC, vfp_fpscr_handler); 323 ci->ci_vfp_id = 0; 324 return; 325 } 326 327 ci->ci_vfp_id = fpsid; 328 switch (fpsid & ~ VFP_FPSID_REV_MSK) { 329 case FPU_VFP10_ARM10E: 330 model = "VFP10 R1"; 331 break; 332 case FPU_VFP11_ARM11: 333 model = "VFP11"; 334 break; 335 case FPU_VFP_MV88SV58XX: 336 model = "VFP3"; 337 break; 338 case FPU_VFP_CORTEXA5: 339 case FPU_VFP_CORTEXA7: 340 case FPU_VFP_CORTEXA8: 341 case FPU_VFP_CORTEXA9: 342 case FPU_VFP_CORTEXA12: 343 case FPU_VFP_CORTEXA15: 344 case FPU_VFP_CORTEXA15_QEMU: 345 case FPU_VFP_CORTEXA17: 346 case FPU_VFP_CORTEXA53: 347 case FPU_VFP_CORTEXA57: 348 if (armreg_cpacr_read() & CPACR_V7_ASEDIS) { 349 model = "VFP 4.0+"; 350 } else { 351 model = "NEON MPE (VFP 3.0+)"; 352 cpu_neon_present = 1; 353 } 354 break; 355 default: 356 aprint_normal_dev(ci->ci_dev, "unrecognized VFP version %#x\n", 357 fpsid); 358 if (CPU_IS_PRIMARY(ci)) 359 install_coproc_handler(VFP_COPROC, vfp_fpscr_handler); 360 vfp_fpscr_changable = VFP_FPSCR_CSUM|VFP_FPSCR_ESUM 361 |VFP_FPSCR_RMODE; 362 vfp_fpscr_default = 0; 363 return; 364 } 365 366 cpu_fpu_present = 1; 367 cpu_media_and_vfp_features[0] = armreg_mvfr0_read(); 368 cpu_media_and_vfp_features[1] = armreg_mvfr1_read(); 369 if (fpsid != 0) { 370 uint32_t f0 = armreg_mvfr0_read(); 371 uint32_t f1 = armreg_mvfr1_read(); 372 aprint_normal("vfp%d at %s: %s%s%s%s%s\n", 373 device_unit(ci->ci_dev), 374 device_xname(ci->ci_dev), 375 model, 376 ((f0 & ARM_MVFR0_ROUNDING_MASK) ? ", rounding" : ""), 377 ((f0 & ARM_MVFR0_EXCEPT_MASK) ? ", exceptions" : ""), 378 ((f1 & ARM_MVFR1_D_NAN_MASK) ? ", NaN propagation" : ""), 379 ((f1 & ARM_MVFR1_FTZ_MASK) ? ", denormals" : "")); 380 aprint_debug("vfp%d: mvfr: [0]=%#x [1]=%#x\n", 381 device_unit(ci->ci_dev), f0, f1); 382 if (CPU_IS_PRIMARY(ci)) { 383 if (f0 & ARM_MVFR0_ROUNDING_MASK) { 384 vfp_fpscr_changable |= VFP_FPSCR_RMODE; 385 } 386 if (f1 & ARM_MVFR0_EXCEPT_MASK) { 387 vfp_fpscr_changable |= VFP_FPSCR_ESUM; 388 } 389 // If hardware supports propagation of NaNs, select it. 390 if (f1 & ARM_MVFR1_D_NAN_MASK) { 391 vfp_fpscr_default &= ~VFP_FPSCR_DN; 392 vfp_fpscr_changable |= VFP_FPSCR_DN; 393 } 394 // If hardware supports denormalized numbers, use it. 395 if (cpu_media_and_vfp_features[1] & ARM_MVFR1_FTZ_MASK) { 396 vfp_fpscr_default &= ~VFP_FPSCR_FZ; 397 vfp_fpscr_changable |= VFP_FPSCR_FZ; 398 } 399 } 400 } 401 evcnt_attach_dynamic(&ci->ci_vfp_evs[0], EVCNT_TYPE_MISC, NULL, 402 ci->ci_cpuname, "vfp coproc use"); 403 evcnt_attach_dynamic(&ci->ci_vfp_evs[1], EVCNT_TYPE_MISC, NULL, 404 ci->ci_cpuname, "vfp coproc re-use"); 405 evcnt_attach_dynamic(&ci->ci_vfp_evs[2], EVCNT_TYPE_TRAP, NULL, 406 ci->ci_cpuname, "vfp coproc fault"); 407 if (CPU_IS_PRIMARY(ci)) { 408 install_coproc_handler(VFP_COPROC, vfp_handler); 409 install_coproc_handler(VFP_COPROC2, vfp_handler); 410 #ifdef CPU_CORTEX 411 if (cpu_neon_present) { 412 install_coproc_handler(CORE_UNKNOWN_HANDLER, 413 neon_handler); 414 aes_md_init(&aes_neon_impl); 415 chacha_md_init(&chacha_neon_impl); 416 } 417 #endif 418 } 419 } 420 421 /* The real handler for VFP bounces. */ 422 static int 423 vfp_handler(u_int address, u_int insn, trapframe_t *frame, int fault_code) 424 { 425 struct cpu_info * const ci = curcpu(); 426 uint32_t fpexc; 427 428 /* This shouldn't ever happen. */ 429 if (fault_code != FAULT_USER && 430 (curlwp->l_flag & (LW_SYSTEM|LW_SYSTEM_FPU)) == LW_SYSTEM) 431 panic("VFP fault at %#x in non-user mode", frame->tf_pc); 432 433 if (ci->ci_vfp_id == 0) { 434 /* No VFP detected, just fault. */ 435 return 1; 436 } 437 438 /* 439 * If we already own the FPU and it's enabled (and no exception), raise 440 * SIGILL. If there is an exception, raise SIGFPE. 441 */ 442 if (curlwp->l_pcu_cpu[PCU_FPU] == ci) { 443 KASSERT(ci->ci_pcu_curlwp[PCU_FPU] == curlwp); 444 445 fpexc = armreg_fpexc_read(); 446 if (fpexc & VFP_FPEXC_EN) { 447 if ((fpexc & VFP_FPEXC_EX) == 0) { 448 return 1; /* SIGILL */ 449 } else { 450 goto fpe; /* SIGFPE; skip pcu_load(9) */ 451 } 452 } 453 } 454 455 /* 456 * Make sure we own the FP. 457 */ 458 pcu_load(&arm_vfp_ops); 459 460 fpexc = armreg_fpexc_read(); 461 if (fpexc & VFP_FPEXC_EX) { 462 ksiginfo_t ksi; 463 KASSERT(fpexc & VFP_FPEXC_EN); 464 465 fpe: 466 curcpu()->ci_vfp_evs[2].ev_count++; 467 468 /* 469 * Need the clear the exception condition so any signal 470 * and future use can proceed. 471 */ 472 armreg_fpexc_write(fpexc & ~(VFP_FPEXC_EX|VFP_FPEXC_FSUM)); 473 474 pcu_save(&arm_vfp_ops, curlwp); 475 476 /* 477 * XXX Need to emulate bounce instructions here to get correct 478 * XXX exception codes, etc. 479 */ 480 KSI_INIT_TRAP(&ksi); 481 ksi.ksi_signo = SIGFPE; 482 if (fpexc & VFP_FPEXC_IXF) 483 ksi.ksi_code = FPE_FLTRES; 484 else if (fpexc & VFP_FPEXC_UFF) 485 ksi.ksi_code = FPE_FLTUND; 486 else if (fpexc & VFP_FPEXC_OFF) 487 ksi.ksi_code = FPE_FLTOVF; 488 else if (fpexc & VFP_FPEXC_DZF) 489 ksi.ksi_code = FPE_FLTDIV; 490 else if (fpexc & VFP_FPEXC_IOF) 491 ksi.ksi_code = FPE_FLTINV; 492 ksi.ksi_addr = (uint32_t *)address; 493 ksi.ksi_trap = 0; 494 trapsignal(curlwp, &ksi); 495 return 0; 496 } 497 498 /* Need to restart the faulted instruction. */ 499 // frame->tf_pc -= INSN_SIZE; 500 return 0; 501 } 502 503 #ifdef CPU_CORTEX 504 /* The real handler for NEON bounces. */ 505 static int 506 neon_handler(u_int address, u_int insn, trapframe_t *frame, int fault_code) 507 { 508 struct cpu_info * const ci = curcpu(); 509 510 if (ci->ci_vfp_id == 0) 511 /* No VFP detected, just fault. */ 512 return 1; 513 514 if ((insn & 0xfe000000) != 0xf2000000 515 && (insn & 0xfe000000) != 0xf4000000) 516 /* Not NEON instruction, just fault. */ 517 return 1; 518 519 /* This shouldn't ever happen. */ 520 if (fault_code != FAULT_USER && 521 (curlwp->l_flag & (LW_SYSTEM|LW_SYSTEM_FPU)) == LW_SYSTEM) 522 panic("NEON fault in non-user mode"); 523 524 /* if we already own the FPU and it's enabled, raise SIGILL */ 525 if (curcpu()->ci_pcu_curlwp[PCU_FPU] == curlwp 526 && (armreg_fpexc_read() & VFP_FPEXC_EN) != 0) 527 return 1; 528 529 pcu_load(&arm_vfp_ops); 530 531 /* Need to restart the faulted instruction. */ 532 // frame->tf_pc -= INSN_SIZE; 533 return 0; 534 } 535 #endif 536 537 static void 538 vfp_state_load(lwp_t *l, u_int flags) 539 { 540 struct pcb * const pcb = lwp_getpcb(l); 541 struct vfpreg * const fregs = &pcb->pcb_vfp; 542 543 /* 544 * Instrument VFP usage -- if a process has not previously 545 * used the VFP, mark it as having used VFP for the first time, 546 * and count this event. 547 * 548 * If a process has used the VFP, count a "used VFP, and took 549 * a trap to use it again" event. 550 */ 551 if (__predict_false((flags & PCU_VALID) == 0)) { 552 curcpu()->ci_vfp_evs[0].ev_count++; 553 pcb->pcb_vfp.vfp_fpscr = vfp_fpscr_default; 554 } else { 555 curcpu()->ci_vfp_evs[1].ev_count++; 556 } 557 558 KASSERT((armreg_fpexc_read() & VFP_FPEXC_EN) == 0); 559 /* 560 * If the VFP is already enabled we must be bouncing an instruction. 561 */ 562 if (flags & PCU_REENABLE) { 563 uint32_t fpexc = armreg_fpexc_read(); 564 armreg_fpexc_write(fpexc | VFP_FPEXC_EN); 565 fregs->vfp_fpexc |= VFP_FPEXC_EN; 566 return; 567 } 568 KASSERT((fregs->vfp_fpexc & VFP_FPEXC_EN) == 0); 569 570 /* 571 * Load and Enable the VFP (so that we can write the registers). 572 */ 573 fregs->vfp_fpexc |= VFP_FPEXC_EN; 574 armreg_fpexc_write(fregs->vfp_fpexc); 575 KASSERT(curcpu()->ci_pcu_curlwp[PCU_FPU] == NULL); 576 KASSERT(l->l_pcu_cpu[PCU_FPU] == NULL); 577 578 load_vfpregs(fregs); 579 armreg_fpscr_write(fregs->vfp_fpscr); 580 581 if (fregs->vfp_fpexc & VFP_FPEXC_EX) { 582 /* Need to restore the exception handling state. */ 583 armreg_fpinst_write(fregs->vfp_fpinst); 584 if (fregs->vfp_fpexc & VFP_FPEXC_FP2V) 585 armreg_fpinst2_write(fregs->vfp_fpinst2); 586 } 587 } 588 589 void 590 vfp_state_save(lwp_t *l) 591 { 592 struct pcb * const pcb = lwp_getpcb(l); 593 struct vfpreg * const fregs = &pcb->pcb_vfp; 594 uint32_t fpexc = armreg_fpexc_read(); 595 596 KASSERT(curcpu()->ci_pcu_curlwp[PCU_FPU] == l); 597 KASSERT(curcpu() == l->l_pcu_cpu[PCU_FPU]); 598 KASSERT(curlwp == l || curlwp->l_pcu_cpu[PCU_FPU] != curcpu()); 599 /* 600 * Enable the VFP (so we can read the registers). 601 * Make sure the exception bit is cleared so that we can 602 * safely dump the registers. 603 */ 604 armreg_fpexc_write((fpexc | VFP_FPEXC_EN) & ~VFP_FPEXC_EX); 605 606 fregs->vfp_fpexc = fpexc; 607 if (fpexc & VFP_FPEXC_EX) { 608 /* Need to save the exception handling state */ 609 fregs->vfp_fpinst = armreg_fpinst_read(); 610 if (fpexc & VFP_FPEXC_FP2V) 611 fregs->vfp_fpinst2 = armreg_fpinst2_read(); 612 } 613 fregs->vfp_fpscr = armreg_fpscr_read(); 614 save_vfpregs(fregs); 615 616 /* Disable the VFP. */ 617 armreg_fpexc_write(fpexc & ~VFP_FPEXC_EN); 618 } 619 620 void 621 vfp_state_release(lwp_t *l) 622 { 623 struct pcb * const pcb = lwp_getpcb(l); 624 625 /* 626 * Now mark the VFP as disabled (and our state 627 * has been already saved or is being discarded). 628 */ 629 pcb->pcb_vfp.vfp_fpexc &= ~VFP_FPEXC_EN; 630 631 /* 632 * Turn off the FPU so the next time a VFP instruction is issued 633 * an exception happens. We don't know if this LWP's state was 634 * loaded but if we turned off the FPU for some other LWP, when 635 * pcu_load invokes vfp_state_load it will see that VFP_FPEXC_EN 636 * is still set so it just restore fpexc and return since its 637 * contents are still sitting in the VFP. 638 */ 639 armreg_fpexc_write(armreg_fpexc_read() & ~VFP_FPEXC_EN); 640 } 641 642 void 643 vfp_savecontext(lwp_t *l) 644 { 645 pcu_save(&arm_vfp_ops, l); 646 } 647 648 void 649 vfp_discardcontext(lwp_t *l, bool used_p) 650 { 651 pcu_discard(&arm_vfp_ops, l, used_p); 652 } 653 654 bool 655 vfp_used_p(const lwp_t *l) 656 { 657 return pcu_valid_p(&arm_vfp_ops, l); 658 } 659 660 void 661 vfp_getcontext(struct lwp *l, mcontext_t *mcp, int *flagsp) 662 { 663 if (vfp_used_p(l)) { 664 const struct pcb * const pcb = lwp_getpcb(l); 665 666 pcu_save(&arm_vfp_ops, l); 667 mcp->__fpu.__vfpregs.__vfp_fpscr = pcb->pcb_vfp.vfp_fpscr; 668 memcpy(mcp->__fpu.__vfpregs.__vfp_fstmx, pcb->pcb_vfp.vfp_regs, 669 sizeof(mcp->__fpu.__vfpregs.__vfp_fstmx)); 670 *flagsp |= _UC_FPU|_UC_ARM_VFP; 671 } 672 } 673 674 void 675 vfp_setcontext(struct lwp *l, const mcontext_t *mcp) 676 { 677 struct pcb * const pcb = lwp_getpcb(l); 678 679 pcu_discard(&arm_vfp_ops, l, true); 680 pcb->pcb_vfp.vfp_fpscr = mcp->__fpu.__vfpregs.__vfp_fpscr; 681 memcpy(pcb->pcb_vfp.vfp_regs, mcp->__fpu.__vfpregs.__vfp_fstmx, 682 sizeof(mcp->__fpu.__vfpregs.__vfp_fstmx)); 683 } 684 685 /* 686 * True if this is a system thread with its own private FPU state. 687 */ 688 static inline bool 689 lwp_system_fpu_p(struct lwp *l) 690 { 691 692 return (l->l_flag & (LW_SYSTEM|LW_SYSTEM_FPU)) == 693 (LW_SYSTEM|LW_SYSTEM_FPU); 694 } 695 696 static const struct vfpreg zero_vfpreg; 697 698 void 699 fpu_kern_enter(void) 700 { 701 struct cpu_info *ci; 702 uint32_t fpexc; 703 int s; 704 705 if (lwp_system_fpu_p(curlwp) && !cpu_intr_p()) { 706 KASSERT(!cpu_softintr_p()); 707 return; 708 } 709 710 /* 711 * Block interrupts up to IPL_VM. We must block preemption 712 * since -- if this is a user thread -- there is nowhere to 713 * save the kernel fpu state, and if we want this to be usable 714 * in interrupts, we can't let interrupts interfere with the 715 * fpu state in use since there's nowhere for them to save it. 716 */ 717 s = splvm(); 718 ci = curcpu(); 719 KASSERTMSG(ci->ci_cpl <= IPL_VM, "cpl=%d", ci->ci_cpl); 720 KASSERT(ci->ci_kfpu_spl == -1); 721 ci->ci_kfpu_spl = s; 722 723 /* Save any fpu state on the current CPU. */ 724 pcu_save_all_on_cpu(); 725 726 /* Enable the fpu. */ 727 fpexc = armreg_fpexc_read(); 728 fpexc |= VFP_FPEXC_EN; 729 fpexc &= ~VFP_FPEXC_EX; 730 armreg_fpexc_write(fpexc); 731 } 732 733 void 734 fpu_kern_leave(void) 735 { 736 struct cpu_info *ci = curcpu(); 737 int s; 738 uint32_t fpexc; 739 740 if (lwp_system_fpu_p(curlwp) && !cpu_intr_p()) { 741 KASSERT(!cpu_softintr_p()); 742 return; 743 } 744 745 KASSERT(ci->ci_cpl == IPL_VM); 746 KASSERT(ci->ci_kfpu_spl != -1); 747 748 /* 749 * Zero the fpu registers; otherwise we might leak secrets 750 * through Spectre-class attacks to userland, even if there are 751 * no bugs in fpu state management. 752 */ 753 load_vfpregs(&zero_vfpreg); 754 755 /* 756 * Disable the fpu so that the kernel can't accidentally use 757 * it again. 758 */ 759 fpexc = armreg_fpexc_read(); 760 fpexc &= ~VFP_FPEXC_EN; 761 armreg_fpexc_write(fpexc); 762 763 /* Restore interrupts. */ 764 s = ci->ci_kfpu_spl; 765 ci->ci_kfpu_spl = -1; 766 splx(s); 767 } 768 769 void 770 kthread_fpu_enter_md(void) 771 { 772 773 pcu_load(&arm_vfp_ops); 774 } 775 776 void 777 kthread_fpu_exit_md(void) 778 { 779 780 /* XXX Should vfp_state_release zero the registers itself? */ 781 load_vfpregs(&zero_vfpreg); 782 vfp_discardcontext(curlwp, 0); 783 } 784 785 #endif /* FPU_VFP */ 786