Lines Matching +full:no +full:- +full:chacha
22 * ARE DISCLAIMED. IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY
57 #include <crypto/chacha/arch/arm/chacha_neon.h>
58 #include <crypto/chacha/chacha_impl.h>
69 /* FLDMD <X>, {d0-d15} */
74 __asm __volatile(".fpu vfp\n vldmia\t%0, {d0-d15}" :: "r" (p) : "memory"); in load_vfpregs_lo()
77 /* FSTMD <X>, {d0-d15} */
82 __asm __volatile(".fpu vfp\n vstmia\t%0, {d0-d15}" :: "r" (p) : "memory"); in save_vfpregs_lo()
86 /* FLDMD <X>, {d16-d31} */
91 __asm __volatile(".fpu neon-vfpv4\n vldmia\t%0, {d16-d31}" :: "r" (&p[16]) : "memory"); in load_vfpregs_hi()
94 /* FLDMD <X>, {d16-d31} */
99 __asm __volatile(".fpu neon-vfpv4\nvstmia\t%0, {d16-d31}" :: "r" (&p[16]) : "memory"); in save_vfpregs_hi()
106 load_vfpregs_lo(fregs->vfp_regs); in load_vfpregs()
109 switch (curcpu()->ci_vfp_id) { in load_vfpregs()
119 load_vfpregs_hi(fregs->vfp_regs); in load_vfpregs()
130 save_vfpregs_lo(fregs->vfp_regs); in save_vfpregs()
133 switch (curcpu()->ci_vfp_id) { in save_vfpregs()
143 save_vfpregs_hi(fregs->vfp_regs); in save_vfpregs()
198 if (pcb->pcb_vfp.vfp_fpexc & VFP_FPEXC_EN) in vfp_fpscr_handler()
202 pcb->pcb_vfp.vfp_fpscr = vfp_fpscr_default; in vfp_fpscr_handler()
209 register_t * const regp = &frame->tf_r0 + regno; in vfp_fpscr_handler()
211 *regp = pcb->pcb_vfp.vfp_fpscr; in vfp_fpscr_handler()
213 pcb->pcb_vfp.vfp_fpscr &= ~vfp_fpscr_changable; in vfp_fpscr_handler()
214 pcb->pcb_vfp.vfp_fpscr |= *regp & vfp_fpscr_changable; in vfp_fpscr_handler()
217 curcpu()->ci_vfp_evs[0].ev_count++; in vfp_fpscr_handler()
219 frame->tf_pc += INSN_SIZE; in vfp_fpscr_handler()
227 ci->ci_vfp_id = 0; in vfp_detect()
240 evcnt_attach_dynamic(&ci->ci_vfp_evs[0], EVCNT_TYPE_TRAP, NULL, in vfp_attach()
241 ci->ci_cpuname, "vfp fpscr traps"); in vfp_attach()
249 if (CPU_ID_ARM11_P(ci->ci_arm_cpuid) in vfp_detect()
250 || CPU_ID_MV88SV58XX_P(ci->ci_arm_cpuid) in vfp_detect()
251 || CPU_ID_CORTEX_P(ci->ci_arm_cpuid)) { in vfp_detect()
256 ci->ci_fp_id = 0; in vfp_detect()
280 ci->ci_vfp_id = 0; in vfp_detect()
286 ci->ci_vfp_id = -1; in vfp_detect()
289 if (ci->ci_vfp_id == 0) { in vfp_detect()
293 ci->ci_vfp_id = fpsid; in vfp_detect()
295 ci->ci_mvfr[0] = armreg_mvfr0_read(); in vfp_detect()
296 ci->ci_mvfr[1] = armreg_mvfr1_read(); in vfp_detect()
305 switch (ci->ci_vfp_id & ~ VFP_FPSID_REV_MSK) { in vfp_attach()
333 aprint_normal_dev(ci->ci_dev, "unrecognized VFP version %#x\n", in vfp_attach()
334 ci->ci_vfp_id); in vfp_attach()
345 const uint32_t f0 = ci->ci_mvfr[0]; in vfp_attach()
346 const uint32_t f1 = ci->ci_mvfr[1]; in vfp_attach()
348 device_unit(ci->ci_dev), in vfp_attach()
349 device_xname(ci->ci_dev), in vfp_attach()
357 device_unit(ci->ci_dev), f0, f1); in vfp_attach()
392 evcnt_attach_dynamic(&ci->ci_vfp_evs[0], EVCNT_TYPE_MISC, NULL, in vfp_attach()
393 ci->ci_cpuname, "vfp coproc use"); in vfp_attach()
394 evcnt_attach_dynamic(&ci->ci_vfp_evs[1], EVCNT_TYPE_MISC, NULL, in vfp_attach()
395 ci->ci_cpuname, "vfp coproc re-use"); in vfp_attach()
396 evcnt_attach_dynamic(&ci->ci_vfp_evs[2], EVCNT_TYPE_TRAP, NULL, in vfp_attach()
397 ci->ci_cpuname, "vfp coproc fault"); in vfp_attach()
409 (curlwp->l_flag & (LW_SYSTEM|LW_SYSTEM_FPU)) == LW_SYSTEM) in vfp_handler()
410 panic("VFP fault at %#x in non-user mode", frame->tf_pc); in vfp_handler()
412 if (ci->ci_vfp_id == 0) { in vfp_handler()
413 /* No VFP detected, just fault. */ in vfp_handler()
418 * If we already own the FPU and it's enabled (and no exception), raise in vfp_handler()
421 if (curlwp->l_pcu_cpu[PCU_FPU] == ci) { in vfp_handler()
422 KASSERT(ci->ci_pcu_curlwp[PCU_FPU] == curlwp); in vfp_handler()
445 curcpu()->ci_vfp_evs[2].ev_count++; in vfp_handler()
478 // frame->tf_pc -= INSN_SIZE; in vfp_handler()
489 if (ci->ci_vfp_id == 0) in neon_handler()
490 /* No VFP detected, just fault. */ in neon_handler()
500 (curlwp->l_flag & (LW_SYSTEM|LW_SYSTEM_FPU)) == LW_SYSTEM) in neon_handler()
501 panic("NEON fault in non-user mode"); in neon_handler()
504 if (curcpu()->ci_pcu_curlwp[PCU_FPU] == curlwp in neon_handler()
511 // frame->tf_pc -= INSN_SIZE; in neon_handler()
520 struct vfpreg * const fregs = &pcb->pcb_vfp; in vfp_state_load()
523 * Instrument VFP usage -- if a process has not previously in vfp_state_load()
531 curcpu()->ci_vfp_evs[0].ev_count++; in vfp_state_load()
532 pcb->pcb_vfp.vfp_fpscr = vfp_fpscr_default; in vfp_state_load()
534 curcpu()->ci_vfp_evs[1].ev_count++; in vfp_state_load()
544 fregs->vfp_fpexc |= VFP_FPEXC_EN; in vfp_state_load()
547 KASSERT((fregs->vfp_fpexc & VFP_FPEXC_EN) == 0); in vfp_state_load()
552 fregs->vfp_fpexc |= VFP_FPEXC_EN; in vfp_state_load()
553 armreg_fpexc_write(fregs->vfp_fpexc); in vfp_state_load()
554 KASSERT(curcpu()->ci_pcu_curlwp[PCU_FPU] == NULL); in vfp_state_load()
555 KASSERT(l->l_pcu_cpu[PCU_FPU] == NULL); in vfp_state_load()
558 armreg_fpscr_write(fregs->vfp_fpscr); in vfp_state_load()
560 if (fregs->vfp_fpexc & VFP_FPEXC_EX) { in vfp_state_load()
562 armreg_fpinst_write(fregs->vfp_fpinst); in vfp_state_load()
563 if (fregs->vfp_fpexc & VFP_FPEXC_FP2V) in vfp_state_load()
564 armreg_fpinst2_write(fregs->vfp_fpinst2); in vfp_state_load()
572 struct vfpreg * const fregs = &pcb->pcb_vfp; in vfp_state_save()
575 KASSERT(curcpu()->ci_pcu_curlwp[PCU_FPU] == l); in vfp_state_save()
576 KASSERT(curcpu() == l->l_pcu_cpu[PCU_FPU]); in vfp_state_save()
577 KASSERT(curlwp == l || curlwp->l_pcu_cpu[PCU_FPU] != curcpu()); in vfp_state_save()
585 fregs->vfp_fpexc = fpexc; in vfp_state_save()
588 fregs->vfp_fpinst = armreg_fpinst_read(); in vfp_state_save()
590 fregs->vfp_fpinst2 = armreg_fpinst2_read(); in vfp_state_save()
592 fregs->vfp_fpscr = armreg_fpscr_read(); in vfp_state_save()
608 pcb->pcb_vfp.vfp_fpexc &= ~VFP_FPEXC_EN; in vfp_state_release()
646 mcp->__fpu.__vfpregs.__vfp_fpscr = pcb->pcb_vfp.vfp_fpscr; in vfp_getcontext()
647 memcpy(mcp->__fpu.__vfpregs.__vfp_fstmx, pcb->pcb_vfp.vfp_regs, in vfp_getcontext()
648 sizeof(mcp->__fpu.__vfpregs.__vfp_fstmx)); in vfp_getcontext()
659 pcb->pcb_vfp.vfp_fpscr = mcp->__fpu.__vfpregs.__vfp_fpscr; in vfp_setcontext()
660 memcpy(pcb->pcb_vfp.vfp_regs, mcp->__fpu.__vfpregs.__vfp_fstmx, in vfp_setcontext()
661 sizeof(mcp->__fpu.__vfpregs.__vfp_fstmx)); in vfp_setcontext()
671 return (l->l_flag & (LW_SYSTEM|LW_SYSTEM_FPU)) == in lwp_system_fpu_p()
691 * since -- if this is a user thread -- there is nowhere to in fpu_kern_enter()
704 KASSERTMSG(ci->ci_cpl <= IPL_VM || cold, "cpl=%d", ci->ci_cpl); in fpu_kern_enter()
706 KASSERT(ci->ci_kfpu_spl == -1); in fpu_kern_enter()
707 ci->ci_kfpu_spl = s; in fpu_kern_enter()
737 KASSERT(ci->ci_cpl == IPL_VM || cold); in fpu_kern_leave()
739 KASSERT(ci->ci_kfpu_spl != -1); in fpu_kern_leave()
743 * through Spectre-class attacks to userland, even if there are in fpu_kern_leave()
744 * no bugs in fpu state management. in fpu_kern_leave()
757 s = ci->ci_kfpu_spl; in fpu_kern_leave()
758 ci->ci_kfpu_spl = -1; in fpu_kern_leave()