1 /* $NetBSD: aes_neon_impl.c,v 1.5 2020/10/10 08:24:10 jmcneill Exp $ */
2
3 /*-
4 * Copyright (c) 2020 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 __KERNEL_RCSID(1, "$NetBSD: aes_neon_impl.c,v 1.5 2020/10/10 08:24:10 jmcneill Exp $");
31
32 #include <sys/types.h>
33 #include <sys/proc.h>
34
35 #include <crypto/aes/aes.h>
36 #include <crypto/aes/aes_impl.h>
37 #include <crypto/aes/arch/arm/aes_neon.h>
38
39 #ifdef __aarch64__
40 #include <aarch64/armreg.h>
41 #endif
42
43 #ifdef _KERNEL
44 #ifndef __aarch64__
45 #include <arm/locore.h>
46 #endif
47 #include <arm/fpu.h>
48 #else
49 #include <sys/sysctl.h>
50 #include <stddef.h>
51 #define fpu_kern_enter() ((void)0)
52 #define fpu_kern_leave() ((void)0)
53 #endif
54
55 static void
aes_neon_setenckey_impl(struct aesenc * enc,const uint8_t * key,uint32_t nrounds)56 aes_neon_setenckey_impl(struct aesenc *enc, const uint8_t *key,
57 uint32_t nrounds)
58 {
59
60 fpu_kern_enter();
61 aes_neon_setenckey(enc, key, nrounds);
62 fpu_kern_leave();
63 }
64
65 static void
aes_neon_setdeckey_impl(struct aesdec * dec,const uint8_t * key,uint32_t nrounds)66 aes_neon_setdeckey_impl(struct aesdec *dec, const uint8_t *key,
67 uint32_t nrounds)
68 {
69
70 fpu_kern_enter();
71 aes_neon_setdeckey(dec, key, nrounds);
72 fpu_kern_leave();
73 }
74
75 static void
aes_neon_enc_impl(const struct aesenc * enc,const uint8_t in[static16],uint8_t out[static16],uint32_t nrounds)76 aes_neon_enc_impl(const struct aesenc *enc, const uint8_t in[static 16],
77 uint8_t out[static 16], uint32_t nrounds)
78 {
79
80 fpu_kern_enter();
81 aes_neon_enc(enc, in, out, nrounds);
82 fpu_kern_leave();
83 }
84
85 static void
aes_neon_dec_impl(const struct aesdec * dec,const uint8_t in[static16],uint8_t out[static16],uint32_t nrounds)86 aes_neon_dec_impl(const struct aesdec *dec, const uint8_t in[static 16],
87 uint8_t out[static 16], uint32_t nrounds)
88 {
89
90 fpu_kern_enter();
91 aes_neon_dec(dec, in, out, nrounds);
92 fpu_kern_leave();
93 }
94
95 static void
aes_neon_cbc_enc_impl(const struct aesenc * enc,const uint8_t in[static16],uint8_t out[static16],size_t nbytes,uint8_t iv[static16],uint32_t nrounds)96 aes_neon_cbc_enc_impl(const struct aesenc *enc, const uint8_t in[static 16],
97 uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16],
98 uint32_t nrounds)
99 {
100
101 if (nbytes == 0)
102 return;
103 fpu_kern_enter();
104 aes_neon_cbc_enc(enc, in, out, nbytes, iv, nrounds);
105 fpu_kern_leave();
106 }
107
108 static void
aes_neon_cbc_dec_impl(const struct aesdec * dec,const uint8_t in[static16],uint8_t out[static16],size_t nbytes,uint8_t iv[static16],uint32_t nrounds)109 aes_neon_cbc_dec_impl(const struct aesdec *dec, const uint8_t in[static 16],
110 uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16],
111 uint32_t nrounds)
112 {
113
114 if (nbytes == 0)
115 return;
116 fpu_kern_enter();
117 aes_neon_cbc_dec(dec, in, out, nbytes, iv, nrounds);
118 fpu_kern_leave();
119 }
120
121 static void
aes_neon_xts_enc_impl(const struct aesenc * enc,const uint8_t in[static16],uint8_t out[static16],size_t nbytes,uint8_t iv[static16],uint32_t nrounds)122 aes_neon_xts_enc_impl(const struct aesenc *enc, const uint8_t in[static 16],
123 uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16],
124 uint32_t nrounds)
125 {
126
127 if (nbytes == 0)
128 return;
129 fpu_kern_enter();
130 aes_neon_xts_enc(enc, in, out, nbytes, iv, nrounds);
131 fpu_kern_leave();
132 }
133
134 static void
aes_neon_xts_dec_impl(const struct aesdec * dec,const uint8_t in[static16],uint8_t out[static16],size_t nbytes,uint8_t iv[static16],uint32_t nrounds)135 aes_neon_xts_dec_impl(const struct aesdec *dec, const uint8_t in[static 16],
136 uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16],
137 uint32_t nrounds)
138 {
139
140 if (nbytes == 0)
141 return;
142 fpu_kern_enter();
143 aes_neon_xts_dec(dec, in, out, nbytes, iv, nrounds);
144 fpu_kern_leave();
145 }
146
147 static void
aes_neon_cbcmac_update1_impl(const struct aesenc * enc,const uint8_t in[static16],size_t nbytes,uint8_t auth[static16],uint32_t nrounds)148 aes_neon_cbcmac_update1_impl(const struct aesenc *enc,
149 const uint8_t in[static 16], size_t nbytes, uint8_t auth[static 16],
150 uint32_t nrounds)
151 {
152
153 fpu_kern_enter();
154 aes_neon_cbcmac_update1(enc, in, nbytes, auth, nrounds);
155 fpu_kern_leave();
156 }
157
158 static void
aes_neon_ccm_enc1_impl(const struct aesenc * enc,const uint8_t in[static16],uint8_t out[static16],size_t nbytes,uint8_t authctr[static32],uint32_t nrounds)159 aes_neon_ccm_enc1_impl(const struct aesenc *enc, const uint8_t in[static 16],
160 uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
161 uint32_t nrounds)
162 {
163
164 fpu_kern_enter();
165 aes_neon_ccm_enc1(enc, in, out, nbytes, authctr, nrounds);
166 fpu_kern_leave();
167 }
168
169 static void
aes_neon_ccm_dec1_impl(const struct aesenc * enc,const uint8_t in[static16],uint8_t out[static16],size_t nbytes,uint8_t authctr[static32],uint32_t nrounds)170 aes_neon_ccm_dec1_impl(const struct aesenc *enc, const uint8_t in[static 16],
171 uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
172 uint32_t nrounds)
173 {
174
175 fpu_kern_enter();
176 aes_neon_ccm_dec1(enc, in, out, nbytes, authctr, nrounds);
177 fpu_kern_leave();
178 }
179
180 static int
aes_neon_probe(void)181 aes_neon_probe(void)
182 {
183 #ifdef __aarch64__
184 struct aarch64_sysctl_cpu_id *id;
185 #endif
186 int result = 0;
187
188 /* Verify that the CPU supports NEON. */
189 #ifdef __aarch64__
190 #ifdef _KERNEL
191 id = &curcpu()->ci_id;
192 #else
193 struct aarch64_sysctl_cpu_id ids;
194 size_t idlen;
195 id = &ids;
196 idlen = sizeof ids;
197 if (sysctlbyname("machdep.cpu0.cpu_id", id, &idlen, NULL, 0))
198 return -1;
199 if (idlen != sizeof ids)
200 return -1;
201 #endif
202 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_ADVSIMD)) {
203 case ID_AA64PFR0_EL1_ADV_SIMD_NONE:
204 return -1;
205 default:
206 break;
207 }
208 #else
209 #ifdef _KERNEL
210 if (!cpu_neon_present)
211 return -1;
212 #else
213 int neon;
214 size_t neonlen = sizeof neon;
215 if (0 && sysctlbyname("machdep.neon_present", &neon, &neonlen, NULL, 0))
216 return -1;
217 if (0 && !neon)
218 return -1;
219 #endif
220 #endif
221
222 fpu_kern_enter();
223 result = aes_neon_selftest();
224 fpu_kern_leave();
225
226 return result;
227 }
228
229 struct aes_impl aes_neon_impl = {
230 .ai_name = "ARM NEON vpaes",
231 .ai_probe = aes_neon_probe,
232 .ai_setenckey = aes_neon_setenckey_impl,
233 .ai_setdeckey = aes_neon_setdeckey_impl,
234 .ai_enc = aes_neon_enc_impl,
235 .ai_dec = aes_neon_dec_impl,
236 .ai_cbc_enc = aes_neon_cbc_enc_impl,
237 .ai_cbc_dec = aes_neon_cbc_dec_impl,
238 .ai_xts_enc = aes_neon_xts_enc_impl,
239 .ai_xts_dec = aes_neon_xts_dec_impl,
240 .ai_cbcmac_update1 = aes_neon_cbcmac_update1_impl,
241 .ai_ccm_enc1 = aes_neon_ccm_enc1_impl,
242 .ai_ccm_dec1 = aes_neon_ccm_dec1_impl,
243 };
244