1 /* $NetBSD: vfp_init.c,v 1.78 2022/08/20 11:34:08 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2008 ARM Ltd 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of the company may not be used to endorse or promote 16 * products derived from this software without specific prior written 17 * permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY 23 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 25 * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 28 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 29 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include "opt_cputypes.h" 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: vfp_init.c,v 1.78 2022/08/20 11:34:08 riastradh Exp $"); 36 37 #include <sys/param.h> 38 #include <sys/types.h> 39 #include <sys/systm.h> 40 #include <sys/device.h> 41 #include <sys/kernel.h> 42 #include <sys/kthread.h> 43 #include <sys/proc.h> 44 #include <sys/cpu.h> 45 46 #include <arm/locore.h> 47 #include <arm/pcb.h> 48 #include <arm/undefined.h> 49 #include <arm/vfpreg.h> 50 #include <arm/mcontext.h> 51 #include <arm/fpu.h> 52 53 #include <uvm/uvm_extern.h> /* for pmap.h */ 54 55 #include <crypto/aes/aes_impl.h> 56 #include <crypto/aes/arch/arm/aes_neon.h> 57 #include <crypto/chacha/arch/arm/chacha_neon.h> 58 #include <crypto/chacha/chacha_impl.h> 59 60 #ifdef FPU_VFP 61 62 #ifdef CPU_CORTEX 63 #define SETFPU __asm(".fpu\tvfpv4") 64 #else 65 #define SETFPU __asm(".fpu\tvfp") 66 #endif 67 SETFPU; 68 69 /* FLDMD <X>, {d0-d15} */ 70 static inline void 71 load_vfpregs_lo(const uint64_t *p) 72 { 73 SETFPU; 74 __asm __volatile(".fpu vfp\n vldmia\t%0, {d0-d15}" :: "r" (p) : "memory"); 75 } 76 77 /* FSTMD <X>, {d0-d15} */ 78 static inline void 79 save_vfpregs_lo(uint64_t *p) 80 { 81 SETFPU; 82 __asm __volatile(".fpu vfp\n vstmia\t%0, {d0-d15}" :: "r" (p) : "memory"); 83 } 84 85 #ifdef CPU_CORTEX 86 /* FLDMD <X>, {d16-d31} */ 87 static inline void 88 load_vfpregs_hi(const uint64_t *p) 89 { 90 SETFPU; 91 __asm __volatile(".fpu neon-vfpv4\n vldmia\t%0, {d16-d31}" :: "r" (&p[16]) : "memory"); 92 } 93 94 /* FLDMD <X>, {d16-d31} */ 95 static inline void 96 save_vfpregs_hi(uint64_t *p) 97 { 98 SETFPU; 99 __asm __volatile(".fpu neon-vfpv4\nvstmia\t%0, {d16-d31}" :: "r" (&p[16]) : "memory"); 100 } 101 #endif 102 103 static inline void 104 load_vfpregs(const struct vfpreg *fregs) 105 { 106 load_vfpregs_lo(fregs->vfp_regs); 107 #ifdef CPU_CORTEX 108 #ifdef CPU_ARM11 109 switch (curcpu()->ci_vfp_id) { 110 case FPU_VFP_CORTEXA5: 111 case FPU_VFP_CORTEXA7: 112 case FPU_VFP_CORTEXA8: 113 case FPU_VFP_CORTEXA9: 114 case FPU_VFP_CORTEXA15: 115 case FPU_VFP_CORTEXA15_QEMU: 116 case FPU_VFP_CORTEXA53: 117 case FPU_VFP_CORTEXA57: 118 #endif 119 load_vfpregs_hi(fregs->vfp_regs); 120 #ifdef CPU_ARM11 121 break; 122 } 123 #endif 124 #endif 125 } 126 127 static inline void 128 save_vfpregs(struct vfpreg *fregs) 129 { 130 save_vfpregs_lo(fregs->vfp_regs); 131 #ifdef CPU_CORTEX 132 #ifdef CPU_ARM11 133 switch (curcpu()->ci_vfp_id) { 134 case FPU_VFP_CORTEXA5: 135 case FPU_VFP_CORTEXA7: 136 case FPU_VFP_CORTEXA8: 137 case FPU_VFP_CORTEXA9: 138 case FPU_VFP_CORTEXA15: 139 case FPU_VFP_CORTEXA15_QEMU: 140 case FPU_VFP_CORTEXA53: 141 case FPU_VFP_CORTEXA57: 142 #endif 143 save_vfpregs_hi(fregs->vfp_regs); 144 #ifdef CPU_ARM11 145 break; 146 } 147 #endif 148 #endif 149 } 150 151 /* The real handler for VFP bounces. */ 152 static int vfp_handler(u_int, u_int, trapframe_t *, int); 153 #ifdef CPU_CORTEX 154 static int neon_handler(u_int, u_int, trapframe_t *, int); 155 #endif 156 157 static void vfp_state_load(lwp_t *, u_int); 158 static void vfp_state_save(lwp_t *); 159 static void vfp_state_release(lwp_t *); 160 161 const pcu_ops_t arm_vfp_ops = { 162 .pcu_id = PCU_FPU, 163 .pcu_state_save = vfp_state_save, 164 .pcu_state_load = vfp_state_load, 165 .pcu_state_release = vfp_state_release, 166 }; 167 168 /* determine what bits can be changed */ 169 uint32_t vfp_fpscr_changable = VFP_FPSCR_CSUM; 170 /* default to run fast */ 171 uint32_t vfp_fpscr_default = (VFP_FPSCR_DN | VFP_FPSCR_FZ | VFP_FPSCR_RN); 172 173 #else 174 /* determine what bits can be changed */ 175 uint32_t vfp_fpscr_changable = VFP_FPSCR_CSUM|VFP_FPSCR_ESUM|VFP_FPSCR_RMODE; 176 #endif /* FPU_VFP */ 177 178 static int 179 vfp_fpscr_handler(u_int address, u_int insn, trapframe_t *frame, int fault_code) 180 { 181 struct lwp * const l = curlwp; 182 const u_int regno = (insn >> 12) & 0xf; 183 /* 184 * Only match move to/from the FPSCR register and we 185 * can't be using the SP,LR,PC as a source. 186 */ 187 if ((insn & 0xffef0fff) != 0xeee10a10 || regno > 12) 188 return 1; 189 190 struct pcb * const pcb = lwp_getpcb(l); 191 192 #ifdef FPU_VFP 193 /* 194 * If FPU is valid somewhere, let's just reenable VFP and 195 * retry the instruction (only safe thing to do since the 196 * pcb has a stale copy). 197 */ 198 if (pcb->pcb_vfp.vfp_fpexc & VFP_FPEXC_EN) 199 return 1; 200 201 if (__predict_false(!vfp_used_p(l))) { 202 pcb->pcb_vfp.vfp_fpscr = vfp_fpscr_default; 203 } 204 #endif 205 206 /* 207 * We now know the pcb has the saved copy. 208 */ 209 register_t * const regp = &frame->tf_r0 + regno; 210 if (insn & 0x00100000) { 211 *regp = pcb->pcb_vfp.vfp_fpscr; 212 } else { 213 pcb->pcb_vfp.vfp_fpscr &= ~vfp_fpscr_changable; 214 pcb->pcb_vfp.vfp_fpscr |= *regp & vfp_fpscr_changable; 215 } 216 217 curcpu()->ci_vfp_evs[0].ev_count++; 218 219 frame->tf_pc += INSN_SIZE; 220 return 0; 221 } 222 223 #ifndef FPU_VFP 224 void 225 vfp_detect(struct cpu_info *ci) 226 { 227 ci->ci_vfp_id = 0; 228 return; 229 } 230 /* 231 * If we don't want VFP support, we still need to handle emulating VFP FPSCR 232 * instructions. 233 */ 234 void 235 vfp_attach(struct cpu_info *ci) 236 { 237 if (CPU_IS_PRIMARY(ci)) { 238 replace_coproc_handler(VFP_COPROC, vfp_fpscr_handler); 239 } 240 evcnt_attach_dynamic(&ci->ci_vfp_evs[0], EVCNT_TYPE_TRAP, NULL, 241 ci->ci_cpuname, "vfp fpscr traps"); 242 } 243 244 #else 245 void 246 vfp_detect(struct cpu_info *ci) 247 { 248 249 if (CPU_ID_ARM11_P(ci->ci_arm_cpuid) 250 || CPU_ID_MV88SV58XX_P(ci->ci_arm_cpuid) 251 || CPU_ID_CORTEX_P(ci->ci_arm_cpuid)) { 252 #if 0 253 const uint32_t nsacr = armreg_nsacr_read(); 254 const uint32_t nsacr_vfp = __BITS(VFP_COPROC,VFP_COPROC2); 255 if ((nsacr & nsacr_vfp) != nsacr_vfp) { 256 ci->ci_fp_id = 0; 257 return; 258 } 259 #endif 260 const uint32_t cpacr_vfp = CPACR_CPn(VFP_COPROC); 261 const uint32_t cpacr_vfp2 = CPACR_CPn(VFP_COPROC2); 262 263 /* 264 * We first need to enable access to the coprocessors. 265 */ 266 uint32_t cpacr = armreg_cpacr_read(); 267 cpacr |= __SHIFTIN(CPACR_ALL, cpacr_vfp); 268 cpacr |= __SHIFTIN(CPACR_ALL, cpacr_vfp2); 269 armreg_cpacr_write(cpacr); 270 271 isb(); 272 273 /* 274 * If we could enable them, then they exist. 275 */ 276 cpacr = armreg_cpacr_read(); 277 bool vfp_p = __SHIFTOUT(cpacr, cpacr_vfp2) == CPACR_ALL 278 && __SHIFTOUT(cpacr, cpacr_vfp) == CPACR_ALL; 279 if (!vfp_p) { 280 ci->ci_vfp_id = 0; 281 return; 282 } 283 } 284 285 /* borrow the ci_vfd_id field for VFP detection */ 286 ci->ci_vfp_id = -1; 287 288 const uint32_t fpsid = armreg_fpsid_read(); 289 if (ci->ci_vfp_id == 0) { 290 return; 291 } 292 293 ci->ci_vfp_id = fpsid; 294 295 ci->ci_mvfr[0] = armreg_mvfr0_read(); 296 ci->ci_mvfr[1] = armreg_mvfr1_read(); 297 298 } 299 300 void 301 vfp_attach(struct cpu_info *ci) 302 { 303 const char *model = NULL; 304 305 switch (ci->ci_vfp_id & ~ VFP_FPSID_REV_MSK) { 306 case FPU_VFP10_ARM10E: 307 model = "VFP10 R1"; 308 break; 309 case FPU_VFP11_ARM11: 310 model = "VFP11"; 311 break; 312 case FPU_VFP_MV88SV58XX: 313 model = "VFP3"; 314 break; 315 case FPU_VFP_CORTEXA5: 316 case FPU_VFP_CORTEXA7: 317 case FPU_VFP_CORTEXA8: 318 case FPU_VFP_CORTEXA9: 319 case FPU_VFP_CORTEXA12: 320 case FPU_VFP_CORTEXA15: 321 case FPU_VFP_CORTEXA15_QEMU: 322 case FPU_VFP_CORTEXA17: 323 case FPU_VFP_CORTEXA53: 324 case FPU_VFP_CORTEXA57: 325 if (armreg_cpacr_read() & CPACR_V7_ASEDIS) { 326 model = "VFP 4.0+"; 327 } else { 328 model = "NEON MPE (VFP 3.0+)"; 329 cpu_neon_present = 1; 330 } 331 break; 332 default: 333 aprint_normal_dev(ci->ci_dev, "unrecognized VFP version %#x\n", 334 ci->ci_vfp_id); 335 if (CPU_IS_PRIMARY(ci)) 336 replace_coproc_handler(VFP_COPROC, vfp_fpscr_handler); 337 vfp_fpscr_changable = VFP_FPSCR_CSUM|VFP_FPSCR_ESUM 338 |VFP_FPSCR_RMODE; 339 vfp_fpscr_default = 0; 340 return; 341 } 342 343 cpu_fpu_present = 1; 344 345 const uint32_t f0 = ci->ci_mvfr[0]; 346 const uint32_t f1 = ci->ci_mvfr[1]; 347 aprint_normal("vfp%d at %s: %s%s%s%s%s\n", 348 device_unit(ci->ci_dev), 349 device_xname(ci->ci_dev), 350 model, 351 ((f0 & ARM_MVFR0_ROUNDING_MASK) ? ", rounding" : ""), 352 ((f0 & ARM_MVFR0_EXCEPT_MASK) ? ", exceptions" : ""), 353 ((f1 & ARM_MVFR1_D_NAN_MASK) ? ", NaN propagation" : ""), 354 ((f1 & ARM_MVFR1_FTZ_MASK) ? ", denormals" : "")); 355 356 aprint_debug("vfp%d: mvfr: [0]=%#x [1]=%#x\n", 357 device_unit(ci->ci_dev), f0, f1); 358 359 if (CPU_IS_PRIMARY(ci)) { 360 cpu_media_and_vfp_features[0] = f0; 361 cpu_media_and_vfp_features[1] = f1; 362 363 if (f0 & ARM_MVFR0_ROUNDING_MASK) { 364 vfp_fpscr_changable |= VFP_FPSCR_RMODE; 365 } 366 if (f1 & ARM_MVFR0_EXCEPT_MASK) { 367 vfp_fpscr_changable |= VFP_FPSCR_ESUM; 368 } 369 // If hardware supports propagation of NaNs, select it. 370 if (f1 & ARM_MVFR1_D_NAN_MASK) { 371 vfp_fpscr_default &= ~VFP_FPSCR_DN; 372 vfp_fpscr_changable |= VFP_FPSCR_DN; 373 } 374 // If hardware supports denormalized numbers, use it. 375 if (f1 & ARM_MVFR1_FTZ_MASK) { 376 vfp_fpscr_default &= ~VFP_FPSCR_FZ; 377 vfp_fpscr_changable |= VFP_FPSCR_FZ; 378 } 379 380 replace_coproc_handler(VFP_COPROC, vfp_handler); 381 install_coproc_handler(VFP_COPROC2, vfp_handler); 382 #ifdef CPU_CORTEX 383 if (cpu_neon_present) { 384 install_coproc_handler(CORE_UNKNOWN_HANDLER, 385 neon_handler); 386 aes_md_init(&aes_neon_impl); 387 chacha_md_init(&chacha_neon_impl); 388 } 389 #endif 390 } 391 392 evcnt_attach_dynamic(&ci->ci_vfp_evs[0], EVCNT_TYPE_MISC, NULL, 393 ci->ci_cpuname, "vfp coproc use"); 394 evcnt_attach_dynamic(&ci->ci_vfp_evs[1], EVCNT_TYPE_MISC, NULL, 395 ci->ci_cpuname, "vfp coproc re-use"); 396 evcnt_attach_dynamic(&ci->ci_vfp_evs[2], EVCNT_TYPE_TRAP, NULL, 397 ci->ci_cpuname, "vfp coproc fault"); 398 } 399 400 /* The real handler for VFP bounces. */ 401 static int 402 vfp_handler(u_int address, u_int insn, trapframe_t *frame, int fault_code) 403 { 404 struct cpu_info * const ci = curcpu(); 405 uint32_t fpexc; 406 407 /* This shouldn't ever happen. */ 408 if (fault_code != FAULT_USER && 409 (curlwp->l_flag & (LW_SYSTEM|LW_SYSTEM_FPU)) == LW_SYSTEM) 410 panic("VFP fault at %#x in non-user mode", frame->tf_pc); 411 412 if (ci->ci_vfp_id == 0) { 413 /* No VFP detected, just fault. */ 414 return 1; 415 } 416 417 /* 418 * If we already own the FPU and it's enabled (and no exception), raise 419 * SIGILL. If there is an exception, raise SIGFPE. 420 */ 421 if (curlwp->l_pcu_cpu[PCU_FPU] == ci) { 422 KASSERT(ci->ci_pcu_curlwp[PCU_FPU] == curlwp); 423 424 fpexc = armreg_fpexc_read(); 425 if (fpexc & VFP_FPEXC_EN) { 426 if ((fpexc & VFP_FPEXC_EX) == 0) { 427 return 1; /* SIGILL */ 428 } else { 429 goto fpe; /* SIGFPE; skip pcu_load(9) */ 430 } 431 } 432 } 433 434 /* 435 * Make sure we own the FP. 436 */ 437 pcu_load(&arm_vfp_ops); 438 439 fpexc = armreg_fpexc_read(); 440 if (fpexc & VFP_FPEXC_EX) { 441 ksiginfo_t ksi; 442 KASSERT(fpexc & VFP_FPEXC_EN); 443 444 fpe: 445 curcpu()->ci_vfp_evs[2].ev_count++; 446 447 /* 448 * Need the clear the exception condition so any signal 449 * and future use can proceed. 450 */ 451 armreg_fpexc_write(fpexc & ~(VFP_FPEXC_EX|VFP_FPEXC_FSUM)); 452 453 pcu_save(&arm_vfp_ops, curlwp); 454 455 /* 456 * XXX Need to emulate bounce instructions here to get correct 457 * XXX exception codes, etc. 458 */ 459 KSI_INIT_TRAP(&ksi); 460 ksi.ksi_signo = SIGFPE; 461 if (fpexc & VFP_FPEXC_IXF) 462 ksi.ksi_code = FPE_FLTRES; 463 else if (fpexc & VFP_FPEXC_UFF) 464 ksi.ksi_code = FPE_FLTUND; 465 else if (fpexc & VFP_FPEXC_OFF) 466 ksi.ksi_code = FPE_FLTOVF; 467 else if (fpexc & VFP_FPEXC_DZF) 468 ksi.ksi_code = FPE_FLTDIV; 469 else if (fpexc & VFP_FPEXC_IOF) 470 ksi.ksi_code = FPE_FLTINV; 471 ksi.ksi_addr = (uint32_t *)address; 472 ksi.ksi_trap = 0; 473 trapsignal(curlwp, &ksi); 474 return 0; 475 } 476 477 /* Need to restart the faulted instruction. */ 478 // frame->tf_pc -= INSN_SIZE; 479 return 0; 480 } 481 482 #ifdef CPU_CORTEX 483 /* The real handler for NEON bounces. */ 484 static int 485 neon_handler(u_int address, u_int insn, trapframe_t *frame, int fault_code) 486 { 487 struct cpu_info * const ci = curcpu(); 488 489 if (ci->ci_vfp_id == 0) 490 /* No VFP detected, just fault. */ 491 return 1; 492 493 if ((insn & 0xfe000000) != 0xf2000000 494 && (insn & 0xfe000000) != 0xf4000000) 495 /* Not NEON instruction, just fault. */ 496 return 1; 497 498 /* This shouldn't ever happen. */ 499 if (fault_code != FAULT_USER && 500 (curlwp->l_flag & (LW_SYSTEM|LW_SYSTEM_FPU)) == LW_SYSTEM) 501 panic("NEON fault in non-user mode"); 502 503 /* if we already own the FPU and it's enabled, raise SIGILL */ 504 if (curcpu()->ci_pcu_curlwp[PCU_FPU] == curlwp 505 && (armreg_fpexc_read() & VFP_FPEXC_EN) != 0) 506 return 1; 507 508 pcu_load(&arm_vfp_ops); 509 510 /* Need to restart the faulted instruction. */ 511 // frame->tf_pc -= INSN_SIZE; 512 return 0; 513 } 514 #endif 515 516 static void 517 vfp_state_load(lwp_t *l, u_int flags) 518 { 519 struct pcb * const pcb = lwp_getpcb(l); 520 struct vfpreg * const fregs = &pcb->pcb_vfp; 521 522 /* 523 * Instrument VFP usage -- if a process has not previously 524 * used the VFP, mark it as having used VFP for the first time, 525 * and count this event. 526 * 527 * If a process has used the VFP, count a "used VFP, and took 528 * a trap to use it again" event. 529 */ 530 if (__predict_false((flags & PCU_VALID) == 0)) { 531 curcpu()->ci_vfp_evs[0].ev_count++; 532 pcb->pcb_vfp.vfp_fpscr = vfp_fpscr_default; 533 } else { 534 curcpu()->ci_vfp_evs[1].ev_count++; 535 } 536 537 KASSERT((armreg_fpexc_read() & VFP_FPEXC_EN) == 0); 538 /* 539 * If the VFP is already enabled we must be bouncing an instruction. 540 */ 541 if (flags & PCU_REENABLE) { 542 uint32_t fpexc = armreg_fpexc_read(); 543 armreg_fpexc_write(fpexc | VFP_FPEXC_EN); 544 fregs->vfp_fpexc |= VFP_FPEXC_EN; 545 return; 546 } 547 KASSERT((fregs->vfp_fpexc & VFP_FPEXC_EN) == 0); 548 549 /* 550 * Load and Enable the VFP (so that we can write the registers). 551 */ 552 fregs->vfp_fpexc |= VFP_FPEXC_EN; 553 armreg_fpexc_write(fregs->vfp_fpexc); 554 KASSERT(curcpu()->ci_pcu_curlwp[PCU_FPU] == NULL); 555 KASSERT(l->l_pcu_cpu[PCU_FPU] == NULL); 556 557 load_vfpregs(fregs); 558 armreg_fpscr_write(fregs->vfp_fpscr); 559 560 if (fregs->vfp_fpexc & VFP_FPEXC_EX) { 561 /* Need to restore the exception handling state. */ 562 armreg_fpinst_write(fregs->vfp_fpinst); 563 if (fregs->vfp_fpexc & VFP_FPEXC_FP2V) 564 armreg_fpinst2_write(fregs->vfp_fpinst2); 565 } 566 } 567 568 void 569 vfp_state_save(lwp_t *l) 570 { 571 struct pcb * const pcb = lwp_getpcb(l); 572 struct vfpreg * const fregs = &pcb->pcb_vfp; 573 uint32_t fpexc = armreg_fpexc_read(); 574 575 KASSERT(curcpu()->ci_pcu_curlwp[PCU_FPU] == l); 576 KASSERT(curcpu() == l->l_pcu_cpu[PCU_FPU]); 577 KASSERT(curlwp == l || curlwp->l_pcu_cpu[PCU_FPU] != curcpu()); 578 /* 579 * Enable the VFP (so we can read the registers). 580 * Make sure the exception bit is cleared so that we can 581 * safely dump the registers. 582 */ 583 armreg_fpexc_write((fpexc | VFP_FPEXC_EN) & ~VFP_FPEXC_EX); 584 585 fregs->vfp_fpexc = fpexc; 586 if (fpexc & VFP_FPEXC_EX) { 587 /* Need to save the exception handling state */ 588 fregs->vfp_fpinst = armreg_fpinst_read(); 589 if (fpexc & VFP_FPEXC_FP2V) 590 fregs->vfp_fpinst2 = armreg_fpinst2_read(); 591 } 592 fregs->vfp_fpscr = armreg_fpscr_read(); 593 save_vfpregs(fregs); 594 595 /* Disable the VFP. */ 596 armreg_fpexc_write(fpexc & ~VFP_FPEXC_EN); 597 } 598 599 void 600 vfp_state_release(lwp_t *l) 601 { 602 struct pcb * const pcb = lwp_getpcb(l); 603 604 /* 605 * Now mark the VFP as disabled (and our state 606 * has been already saved or is being discarded). 607 */ 608 pcb->pcb_vfp.vfp_fpexc &= ~VFP_FPEXC_EN; 609 610 /* 611 * Turn off the FPU so the next time a VFP instruction is issued 612 * an exception happens. We don't know if this LWP's state was 613 * loaded but if we turned off the FPU for some other LWP, when 614 * pcu_load invokes vfp_state_load it will see that VFP_FPEXC_EN 615 * is still set so it just restore fpexc and return since its 616 * contents are still sitting in the VFP. 617 */ 618 armreg_fpexc_write(armreg_fpexc_read() & ~VFP_FPEXC_EN); 619 } 620 621 void 622 vfp_savecontext(lwp_t *l) 623 { 624 pcu_save(&arm_vfp_ops, l); 625 } 626 627 void 628 vfp_discardcontext(lwp_t *l, bool used_p) 629 { 630 pcu_discard(&arm_vfp_ops, l, used_p); 631 } 632 633 bool 634 vfp_used_p(const lwp_t *l) 635 { 636 return pcu_valid_p(&arm_vfp_ops, l); 637 } 638 639 void 640 vfp_getcontext(struct lwp *l, mcontext_t *mcp, int *flagsp) 641 { 642 if (vfp_used_p(l)) { 643 const struct pcb * const pcb = lwp_getpcb(l); 644 645 pcu_save(&arm_vfp_ops, l); 646 mcp->__fpu.__vfpregs.__vfp_fpscr = pcb->pcb_vfp.vfp_fpscr; 647 memcpy(mcp->__fpu.__vfpregs.__vfp_fstmx, pcb->pcb_vfp.vfp_regs, 648 sizeof(mcp->__fpu.__vfpregs.__vfp_fstmx)); 649 *flagsp |= _UC_FPU|_UC_ARM_VFP; 650 } 651 } 652 653 void 654 vfp_setcontext(struct lwp *l, const mcontext_t *mcp) 655 { 656 struct pcb * const pcb = lwp_getpcb(l); 657 658 pcu_discard(&arm_vfp_ops, l, true); 659 pcb->pcb_vfp.vfp_fpscr = mcp->__fpu.__vfpregs.__vfp_fpscr; 660 memcpy(pcb->pcb_vfp.vfp_regs, mcp->__fpu.__vfpregs.__vfp_fstmx, 661 sizeof(mcp->__fpu.__vfpregs.__vfp_fstmx)); 662 } 663 664 /* 665 * True if this is a system thread with its own private FPU state. 666 */ 667 static inline bool 668 lwp_system_fpu_p(struct lwp *l) 669 { 670 671 return (l->l_flag & (LW_SYSTEM|LW_SYSTEM_FPU)) == 672 (LW_SYSTEM|LW_SYSTEM_FPU); 673 } 674 675 static const struct vfpreg zero_vfpreg; 676 677 void 678 fpu_kern_enter(void) 679 { 680 struct cpu_info *ci; 681 uint32_t fpexc; 682 int s; 683 684 if (lwp_system_fpu_p(curlwp) && !cpu_intr_p()) { 685 KASSERT(!cpu_softintr_p()); 686 return; 687 } 688 689 /* 690 * Block interrupts up to IPL_VM. We must block preemption 691 * since -- if this is a user thread -- there is nowhere to 692 * save the kernel fpu state, and if we want this to be usable 693 * in interrupts, we can't let interrupts interfere with the 694 * fpu state in use since there's nowhere for them to save it. 695 */ 696 s = splvm(); 697 ci = curcpu(); 698 #if 0 699 /* 700 * Can't assert this because if the caller holds a spin lock at 701 * IPL_VM, and previously held and released a spin lock at 702 * higher IPL, the IPL remains raised above IPL_VM. 703 */ 704 KASSERTMSG(ci->ci_cpl <= IPL_VM || cold, "cpl=%d", ci->ci_cpl); 705 #endif 706 KASSERT(ci->ci_kfpu_spl == -1); 707 ci->ci_kfpu_spl = s; 708 709 /* Save any fpu state on the current CPU. */ 710 pcu_save_all_on_cpu(); 711 712 /* Enable the fpu. */ 713 fpexc = armreg_fpexc_read(); 714 fpexc |= VFP_FPEXC_EN; 715 fpexc &= ~VFP_FPEXC_EX; 716 armreg_fpexc_write(fpexc); 717 } 718 719 void 720 fpu_kern_leave(void) 721 { 722 struct cpu_info *ci = curcpu(); 723 int s; 724 uint32_t fpexc; 725 726 if (lwp_system_fpu_p(curlwp) && !cpu_intr_p()) { 727 KASSERT(!cpu_softintr_p()); 728 return; 729 } 730 731 #if 0 732 /* 733 * Can't assert this because if the caller holds a spin lock at 734 * IPL_VM, and previously held and released a spin lock at 735 * higher IPL, the IPL remains raised above IPL_VM. 736 */ 737 KASSERT(ci->ci_cpl == IPL_VM || cold); 738 #endif 739 KASSERT(ci->ci_kfpu_spl != -1); 740 741 /* 742 * Zero the fpu registers; otherwise we might leak secrets 743 * through Spectre-class attacks to userland, even if there are 744 * no bugs in fpu state management. 745 */ 746 load_vfpregs(&zero_vfpreg); 747 748 /* 749 * Disable the fpu so that the kernel can't accidentally use 750 * it again. 751 */ 752 fpexc = armreg_fpexc_read(); 753 fpexc &= ~VFP_FPEXC_EN; 754 armreg_fpexc_write(fpexc); 755 756 /* Restore interrupts. */ 757 s = ci->ci_kfpu_spl; 758 ci->ci_kfpu_spl = -1; 759 splx(s); 760 } 761 762 void 763 kthread_fpu_enter_md(void) 764 { 765 766 pcu_load(&arm_vfp_ops); 767 } 768 769 void 770 kthread_fpu_exit_md(void) 771 { 772 773 /* XXX Should vfp_state_release zero the registers itself? */ 774 load_vfpregs(&zero_vfpreg); 775 vfp_discardcontext(curlwp, 0); 776 } 777 778 #endif /* FPU_VFP */ 779