1 /* $NetBSD: chacha_neon_impl.c,v 1.2 2020/10/10 08:24:10 jmcneill Exp $ */
2
3 /*-
4 * Copyright (c) 2020 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 __KERNEL_RCSID(1, "$NetBSD: chacha_neon_impl.c,v 1.2 2020/10/10 08:24:10 jmcneill Exp $");
31
32 #include "chacha_neon.h"
33
34 #ifdef __aarch64__
35 #include <aarch64/armreg.h>
36 #endif
37
38 #ifdef _KERNEL
39 #include <sys/proc.h>
40 #ifndef __aarch64__
41 #include <arm/locore.h>
42 #endif
43 #include <arm/fpu.h>
44 #else
45 #include <sys/sysctl.h>
46 #include <stddef.h>
47 #define fpu_kern_enter() ((void)0)
48 #define fpu_kern_leave() ((void)0)
49 #endif
50
51 static void
chacha_core_neon_impl(uint8_t out[restrict static64],const uint8_t in[static16],const uint8_t k[static32],const uint8_t c[static16],unsigned nr)52 chacha_core_neon_impl(uint8_t out[restrict static 64],
53 const uint8_t in[static 16],
54 const uint8_t k[static 32],
55 const uint8_t c[static 16],
56 unsigned nr)
57 {
58
59 fpu_kern_enter();
60 chacha_core_neon(out, in, k, c, nr);
61 fpu_kern_leave();
62 }
63
64 static void
hchacha_neon_impl(uint8_t out[restrict static32],const uint8_t in[static16],const uint8_t k[static32],const uint8_t c[static16],unsigned nr)65 hchacha_neon_impl(uint8_t out[restrict static 32],
66 const uint8_t in[static 16],
67 const uint8_t k[static 32],
68 const uint8_t c[static 16],
69 unsigned nr)
70 {
71
72 fpu_kern_enter();
73 hchacha_neon(out, in, k, c, nr);
74 fpu_kern_leave();
75 }
76
77 static void
chacha_stream_neon_impl(uint8_t * restrict s,size_t nbytes,uint32_t blkno,const uint8_t nonce[static12],const uint8_t key[static32],unsigned nr)78 chacha_stream_neon_impl(uint8_t *restrict s, size_t nbytes, uint32_t blkno,
79 const uint8_t nonce[static 12],
80 const uint8_t key[static 32],
81 unsigned nr)
82 {
83
84 fpu_kern_enter();
85 chacha_stream_neon(s, nbytes, blkno, nonce, key, nr);
86 fpu_kern_leave();
87 }
88
89 static void
chacha_stream_xor_neon_impl(uint8_t * c,const uint8_t * p,size_t nbytes,uint32_t blkno,const uint8_t nonce[static12],const uint8_t key[static32],unsigned nr)90 chacha_stream_xor_neon_impl(uint8_t *c, const uint8_t *p, size_t nbytes,
91 uint32_t blkno,
92 const uint8_t nonce[static 12],
93 const uint8_t key[static 32],
94 unsigned nr)
95 {
96
97 fpu_kern_enter();
98 chacha_stream_xor_neon(c, p, nbytes, blkno, nonce, key, nr);
99 fpu_kern_leave();
100 }
101
102 static void
xchacha_stream_neon_impl(uint8_t * restrict s,size_t nbytes,uint32_t blkno,const uint8_t nonce[static24],const uint8_t key[static32],unsigned nr)103 xchacha_stream_neon_impl(uint8_t *restrict s, size_t nbytes, uint32_t blkno,
104 const uint8_t nonce[static 24],
105 const uint8_t key[static 32],
106 unsigned nr)
107 {
108
109 fpu_kern_enter();
110 xchacha_stream_neon(s, nbytes, blkno, nonce, key, nr);
111 fpu_kern_leave();
112 }
113
114 static void
xchacha_stream_xor_neon_impl(uint8_t * c,const uint8_t * p,size_t nbytes,uint32_t blkno,const uint8_t nonce[static24],const uint8_t key[static32],unsigned nr)115 xchacha_stream_xor_neon_impl(uint8_t *c, const uint8_t *p, size_t nbytes,
116 uint32_t blkno,
117 const uint8_t nonce[static 24],
118 const uint8_t key[static 32],
119 unsigned nr)
120 {
121
122 fpu_kern_enter();
123 xchacha_stream_xor_neon(c, p, nbytes, blkno, nonce, key, nr);
124 fpu_kern_leave();
125 }
126
127 static int
chacha_probe_neon(void)128 chacha_probe_neon(void)
129 {
130 #ifdef __aarch64__
131 struct aarch64_sysctl_cpu_id *id;
132 #endif
133 int result = 0;
134
135 /* Verify that the CPU supports NEON. */
136 #ifdef __aarch64__
137 #ifdef _KERNEL
138 id = &curcpu()->ci_id;
139 #else
140 struct aarch64_sysctl_cpu_id ids;
141 size_t idlen;
142 id = &ids;
143 idlen = sizeof ids;
144 if (sysctlbyname("machdep.cpu0.cpu_id", id, &idlen, NULL, 0))
145 return -1;
146 if (idlen != sizeof ids)
147 return -1;
148 #endif
149 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_ADVSIMD)) {
150 case ID_AA64PFR0_EL1_ADV_SIMD_NONE:
151 return -1;
152 default:
153 break;
154 }
155 #else
156 #ifdef _KERNEL
157 if (!cpu_neon_present)
158 return -1;
159 #else
160 int neon;
161 size_t neonlen = sizeof neon;
162 if (sysctlbyname("machdep.neon_present", &neon, &neonlen, NULL, 0))
163 return -1;
164 if (!neon)
165 return -1;
166 #endif
167 #endif
168
169 return result;
170 }
171
172 const struct chacha_impl chacha_neon_impl = {
173 .ci_name = "ARM NEON ChaCha",
174 .ci_probe = chacha_probe_neon,
175 .ci_chacha_core = chacha_core_neon_impl,
176 .ci_hchacha = hchacha_neon_impl,
177 .ci_chacha_stream = chacha_stream_neon_impl,
178 .ci_chacha_stream_xor = chacha_stream_xor_neon_impl,
179 .ci_xchacha_stream = xchacha_stream_neon_impl,
180 .ci_xchacha_stream_xor = xchacha_stream_xor_neon_impl,
181 };
182