1 /* $NetBSD: chacha_sse2_impl.c,v 1.1 2020/07/25 22:49:20 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 2020 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 __KERNEL_RCSID(1, "$NetBSD: chacha_sse2_impl.c,v 1.1 2020/07/25 22:49:20 riastradh Exp $");
31
32 #include "chacha_sse2.h"
33
34 #ifdef _KERNEL
35 #include <x86/cpu.h>
36 #include <x86/fpu.h>
37 #else
38 #include <sys/sysctl.h>
39 #include <cpuid.h>
40 #include <stddef.h>
41 #define fpu_kern_enter() ((void)0)
42 #define fpu_kern_leave() ((void)0)
43 #endif
44
45 static void
chacha_core_sse2_impl(uint8_t out[restrict static64],const uint8_t in[static16],const uint8_t k[static32],const uint8_t c[static16],unsigned nr)46 chacha_core_sse2_impl(uint8_t out[restrict static 64],
47 const uint8_t in[static 16],
48 const uint8_t k[static 32],
49 const uint8_t c[static 16],
50 unsigned nr)
51 {
52
53 fpu_kern_enter();
54 chacha_core_sse2(out, in, k, c, nr);
55 fpu_kern_leave();
56 }
57
58 static void
hchacha_sse2_impl(uint8_t out[restrict static32],const uint8_t in[static16],const uint8_t k[static32],const uint8_t c[static16],unsigned nr)59 hchacha_sse2_impl(uint8_t out[restrict static 32],
60 const uint8_t in[static 16],
61 const uint8_t k[static 32],
62 const uint8_t c[static 16],
63 unsigned nr)
64 {
65
66 fpu_kern_enter();
67 hchacha_sse2(out, in, k, c, nr);
68 fpu_kern_leave();
69 }
70
71 static void
chacha_stream_sse2_impl(uint8_t * restrict s,size_t nbytes,uint32_t blkno,const uint8_t nonce[static12],const uint8_t key[static32],unsigned nr)72 chacha_stream_sse2_impl(uint8_t *restrict s, size_t nbytes, uint32_t blkno,
73 const uint8_t nonce[static 12],
74 const uint8_t key[static 32],
75 unsigned nr)
76 {
77
78 fpu_kern_enter();
79 chacha_stream_sse2(s, nbytes, blkno, nonce, key, nr);
80 fpu_kern_leave();
81 }
82
83 static void
chacha_stream_xor_sse2_impl(uint8_t * c,const uint8_t * p,size_t nbytes,uint32_t blkno,const uint8_t nonce[static12],const uint8_t key[static32],unsigned nr)84 chacha_stream_xor_sse2_impl(uint8_t *c, const uint8_t *p, size_t nbytes,
85 uint32_t blkno,
86 const uint8_t nonce[static 12],
87 const uint8_t key[static 32],
88 unsigned nr)
89 {
90
91 fpu_kern_enter();
92 chacha_stream_xor_sse2(c, p, nbytes, blkno, nonce, key, nr);
93 fpu_kern_leave();
94 }
95
96 static void
xchacha_stream_sse2_impl(uint8_t * restrict s,size_t nbytes,uint32_t blkno,const uint8_t nonce[static24],const uint8_t key[static32],unsigned nr)97 xchacha_stream_sse2_impl(uint8_t *restrict s, size_t nbytes, uint32_t blkno,
98 const uint8_t nonce[static 24],
99 const uint8_t key[static 32],
100 unsigned nr)
101 {
102
103 fpu_kern_enter();
104 xchacha_stream_sse2(s, nbytes, blkno, nonce, key, nr);
105 fpu_kern_leave();
106 }
107
108 static void
xchacha_stream_xor_sse2_impl(uint8_t * c,const uint8_t * p,size_t nbytes,uint32_t blkno,const uint8_t nonce[static24],const uint8_t key[static32],unsigned nr)109 xchacha_stream_xor_sse2_impl(uint8_t *c, const uint8_t *p, size_t nbytes,
110 uint32_t blkno,
111 const uint8_t nonce[static 24],
112 const uint8_t key[static 32],
113 unsigned nr)
114 {
115
116 fpu_kern_enter();
117 xchacha_stream_xor_sse2(c, p, nbytes, blkno, nonce, key, nr);
118 fpu_kern_leave();
119 }
120
121 static int
chacha_probe_sse2(void)122 chacha_probe_sse2(void)
123 {
124
125 /* Verify that the CPU supports SSE and SSE2. */
126 #ifdef _KERNEL
127 if (!i386_has_sse)
128 return -1;
129 if (!i386_has_sse2)
130 return -1;
131 #else
132 unsigned eax, ebx, ecx, edx;
133 if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx))
134 return -1;
135 if ((edx & bit_SSE) == 0)
136 return -1;
137 if ((edx & bit_SSE2) == 0)
138 return -1;
139 #endif
140
141 return 0;
142 }
143
144 const struct chacha_impl chacha_sse2_impl = {
145 .ci_name = "x86 SSE2 ChaCha",
146 .ci_probe = chacha_probe_sse2,
147 .ci_chacha_core = chacha_core_sse2_impl,
148 .ci_hchacha = hchacha_sse2_impl,
149 .ci_chacha_stream = chacha_stream_sse2_impl,
150 .ci_chacha_stream_xor = chacha_stream_xor_sse2_impl,
151 .ci_xchacha_stream = xchacha_stream_sse2_impl,
152 .ci_xchacha_stream_xor = xchacha_stream_xor_sse2_impl,
153 };
154