1 /**********************************************************************
2 Copyright(c) 2020 Arm Corporation All rights reserved.
3
4 Redistribution and use in source and binary forms, with or without
5 modification, are permitted provided that the following conditions
6 are met:
7 * Redistributions of source code must retain the above copyright
8 notice, this list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright
10 notice, this list of conditions and the following disclaimer in
11 the documentation and/or other materials provided with the
12 distribution.
13 * Neither the name of Arm Corporation nor the names of its
14 contributors may be used to endorse or promote products derived
15 from this software without specific prior written permission.
16
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 **********************************************************************/
29 #ifndef __AARCH64_MULTIBINARY_H__
30 #define __AARCH64_MULTIBINARY_H__
31 #ifndef __aarch64__
32 #error "This file is for aarch64 only"
33 #endif
34 #include "aarch64_label.h"
35 #ifdef __ASSEMBLY__
36 /**
37 * # mbin_interface : the wrapper layer for isal-l api
38 *
39 * ## references:
40 * * https://sourceware.org/git/gitweb.cgi?p=glibc.git;a=blob;f=sysdeps/aarch64/dl-trampoline.S
41 * * http://infocenter.arm.com/help/topic/com.arm.doc.ihi0055b/IHI0055B_aapcs64.pdf
42 * * https://static.docs.arm.com/ihi0057/b/IHI0057B_aadwarf64.pdf?_ga=2.80574487.1870739014.1564969896-1634778941.1548729310
43 *
44 * ## Usage:
45 * 1. Define dispather function
46 * 2. name must be \name\()_dispatcher
47 * 3. Prototype should be *"void * \name\()_dispatcher"*
48 * 4. The dispather should return the right function pointer , revision and a string information .
49 **/
50 .macro mbin_interface name:req
51 .extern cdecl(\name\()_dispatcher)
52 .data
53 .balign 8
54 .global cdecl(\name\()_dispatcher_info)
55 #ifndef __APPLE__
56 .type \name\()_dispatcher_info,%object
57 #endif
58 cdecl(\name\()_dispatcher_info):
59 .quad \name\()_mbinit //func_entry
60 #ifndef __APPLE__
61 .size \name\()_dispatcher_info,. - \name\()_dispatcher_info
62 #endif
63 .balign 8
64 .text
65 \name\()_mbinit:
66 //save lp fp, sub sp
67 .cfi_startproc
68 stp x29, x30, [sp, -224]!
69
70 //add cfi directive to avoid GDB bt cmds error
71 //set cfi(Call Frame Information)
72 .cfi_def_cfa_offset 224
73 .cfi_offset 29, -224
74 .cfi_offset 30, -216
75
76 //save parameter/result/indirect result registers
77 stp x8, x9, [sp, 16]
78 .cfi_offset 8, -208
79 .cfi_offset 9, -200
80 stp x0, x1, [sp, 32]
81 .cfi_offset 0, -192
82 .cfi_offset 1, -184
83 stp x2, x3, [sp, 48]
84 .cfi_offset 2, -176
85 .cfi_offset 3, -168
86 stp x4, x5, [sp, 64]
87 .cfi_offset 4, -160
88 .cfi_offset 5, -152
89 stp x6, x7, [sp, 80]
90 .cfi_offset 6, -144
91 .cfi_offset 7, -136
92 stp q0, q1, [sp, 96]
93 .cfi_offset 64, -128
94 .cfi_offset 65, -112
95 stp q2, q3, [sp, 128]
96 .cfi_offset 66, -96
97 .cfi_offset 67, -80
98 stp q4, q5, [sp, 160]
99 .cfi_offset 68, -64
100 .cfi_offset 69, -48
101 stp q6, q7, [sp, 192]
102 .cfi_offset 70, -32
103 .cfi_offset 71, -16
104
105 /**
106 * The dispatcher functions have the following prototype:
107 * void * function_dispatcher(void)
108 * As the dispatcher is returning a struct, by the AAPCS,
109 */
110
111
112 bl cdecl(\name\()_dispatcher)
113 //restore temp/indirect result registers
114 ldp x8, x9, [sp, 16]
115 .cfi_restore 8
116 .cfi_restore 9
117
118 // save function entry
119 str x0, [x9]
120
121 //restore parameter/result registers
122 ldp x0, x1, [sp, 32]
123 .cfi_restore 0
124 .cfi_restore 1
125 ldp x2, x3, [sp, 48]
126 .cfi_restore 2
127 .cfi_restore 3
128 ldp x4, x5, [sp, 64]
129 .cfi_restore 4
130 .cfi_restore 5
131 ldp x6, x7, [sp, 80]
132 .cfi_restore 6
133 .cfi_restore 7
134 ldp q0, q1, [sp, 96]
135 .cfi_restore 64
136 .cfi_restore 65
137 ldp q2, q3, [sp, 128]
138 .cfi_restore 66
139 .cfi_restore 67
140 ldp q4, q5, [sp, 160]
141 .cfi_restore 68
142 .cfi_restore 69
143 ldp q6, q7, [sp, 192]
144 .cfi_restore 70
145 .cfi_restore 71
146 //save lp fp and sp
147 ldp x29, x30, [sp], 224
148 //restore cfi setting
149 .cfi_restore 30
150 .cfi_restore 29
151 .cfi_def_cfa_offset 0
152 .cfi_endproc
153
154 .global cdecl(\name)
155 #ifndef __APPLE__
156 .type \name,%function
157 #endif
158 .align 2
159 cdecl(\name\()):
160 #ifndef __APPLE__
161 adrp x9, :got:\name\()_dispatcher_info
162 ldr x9, [x9, #:got_lo12:\name\()_dispatcher_info]
163 #else
164 adrp x9, cdecl(\name\()_dispatcher_info)@GOTPAGE
165 ldr x9, [x9, #cdecl(\name\()_dispatcher_info)@GOTPAGEOFF]
166 #endif
167 ldr x10,[x9]
168 br x10
169 #ifndef __APPLE__
170 .size \name,. - \name
171 #endif
172 .endm
173
174 /**
175 * mbin_interface_base is used for the interfaces which have only
176 * noarch implementation
177 */
178 .macro mbin_interface_base name:req, base:req
179 .extern \base
180 .data
181 .balign 8
182 .global cdecl(\name\()_dispatcher_info)
183 #ifndef __APPLE__
184 .type \name\()_dispatcher_info,%object
185 #endif
186 cdecl(\name\()_dispatcher_info):
187 .quad \base //func_entry
188 #ifndef __APPLE__
189 .size \name\()_dispatcher_info,. - \name\()_dispatcher_info
190 #endif
191 .balign 8
192 .text
193 .global cdecl(\name)
194 #ifndef __APPLE__
195 .type \name,%function
196 #endif
197 .align 2
198 cdecl(\name\()):
199 #ifndef __APPLE__
200 adrp x9, :got:cdecl(_\name\()_dispatcher_info)
201 ldr x9, [x9, #:got_lo12:cdecl(_\name\()_dispatcher_info)]
202 #else
203 adrp x9, cdecl(_\name\()_dispatcher_info)@GOTPAGE
204 ldr x9, [x9, #cdecl(_\name\()_dispatcher_info)@GOTPAGEOFF]
205 #endif
206 ldr x10,[x9]
207 br x10
208 #ifndef __APPLE__
209 .size \name,. - \name
210 #endif
211 .endm
212
213 #else /* __ASSEMBLY__ */
214 #include <stdint.h>
215 #if defined(__linux__)
216 #include <sys/auxv.h>
217 #include <asm/hwcap.h>
218 #elif defined(__APPLE__)
219 #define SYSCTL_PMULL_KEY "hw.optional.arm.FEAT_PMULL" // from macOS 12 FEAT_* sysctl infos are available
220 #define SYSCTL_CRC32_KEY "hw.optional.armv8_crc32"
221 #define SYSCTL_SVE_KEY "hw.optional.arm.FEAT_SVE" // this one is just a guess and need to check macOS update
222 #include <sys/sysctl.h>
223 #include <stddef.h>
sysctlEnabled(const char * name)224 static inline int sysctlEnabled(const char* name){
225 int enabled;
226 size_t size = sizeof(enabled);
227 int status = sysctlbyname(name, &enabled, &size, NULL, 0);
228 return status ? 0 : enabled;
229 }
230 #endif
231
232
233 #define DEFINE_INTERFACE_DISPATCHER(name) \
234 void * name##_dispatcher(void)
235
236 #define PROVIDER_BASIC(name) \
237 PROVIDER_INFO(name##_base)
238
239 #define DO_DIGNOSTIC(x) _Pragma GCC diagnostic ignored "-W"#x
240 #define DO_PRAGMA(x) _Pragma (#x)
241 #define DIGNOSTIC_IGNORE(x) DO_PRAGMA(GCC diagnostic ignored #x)
242 #define DIGNOSTIC_PUSH() DO_PRAGMA(GCC diagnostic push)
243 #define DIGNOSTIC_POP() DO_PRAGMA(GCC diagnostic pop)
244
245
246 #define PROVIDER_INFO(_func_entry) \
247 ({ DIGNOSTIC_PUSH() \
248 DIGNOSTIC_IGNORE(-Wnested-externs) \
249 extern void _func_entry(void); \
250 DIGNOSTIC_POP() \
251 _func_entry; \
252 })
253
254 /**
255 * Micro-Architector definitions
256 * Reference: https://developer.arm.com/docs/ddi0595/f/aarch64-system-registers/midr_el1
257 */
258
259 #define CPU_IMPLEMENTER_RESERVE 0x00
260 #define CPU_IMPLEMENTER_ARM 0x41
261
262
263 #define CPU_PART_CORTEX_A57 0xD07
264 #define CPU_PART_CORTEX_A72 0xD08
265 #define CPU_PART_NEOVERSE_N1 0xD0C
266
267 #define MICRO_ARCH_ID(imp,part) \
268 (((CPU_IMPLEMENTER_##imp&0xff)<<24)|((CPU_PART_##part&0xfff)<<4))
269
270 #ifndef HWCAP_CPUID
271 #define HWCAP_CPUID (1<<11)
272 #endif
273
274 /**
275 * @brief get_micro_arch_id
276 *
277 * read micro-architector register instruction if possible.This function
278 * provides microarchitecture information and make microarchitecture optimization
279 * possible.
280 *
281 * Read system registers(MRS) is forbidden in userspace. If executed, it
282 * will raise illegal instruction error. Kernel provides a solution for
283 * this issue. The solution depends on HWCAP_CPUID flags. Reference(1)
284 * describes how to use it. It provides a "illegal insstruction" handler
285 * in kernel space, the handler will execute MRS and return the correct
286 * value to userspace.
287 *
288 * To avoid too many kernel trap, this function MUST be only called in
289 * dispatcher. And HWCAP must be match,That will make sure there are no
290 * illegal instruction errors. HWCAP_CPUID should be available to get the
291 * best performance.
292 *
293 * NOTICE:
294 * - HWCAP_CPUID should be available. Otherwise it returns reserve value
295 * - It MUST be called inside dispather.
296 * - It MUST meet the HWCAP requirements
297 *
298 * Example:
299 * DEFINE_INTERFACE_DISPATCHER(crc32_iscsi)
300 * {
301 * unsigned long auxval = getauxval(AT_HWCAP);
302 * // MUST do the judgement is MUST.
303 * if ((HWCAP_CRC32 | HWCAP_PMULL) == (auxval & (HWCAP_CRC32 | HWCAP_PMULL))) {
304 * switch (get_micro_arch_id()) {
305 * case MICRO_ARCH_ID(ARM, CORTEX_A57):
306 * return PROVIDER_INFO(crc32_pmull_crc_for_a57);
307 * case MICRO_ARCH_ID(ARM, CORTEX_A72):
308 * return PROVIDER_INFO(crc32_pmull_crc_for_a72);
309 * case MICRO_ARCH_ID(ARM, NEOVERSE_N1):
310 * return PROVIDER_INFO(crc32_pmull_crc_for_n1);
311 * case default:
312 * return PROVIDER_INFO(crc32_pmull_crc_for_others);
313 * }
314 * }
315 * return PROVIDER_BASIC(crc32_iscsi);
316 * }
317 * KNOWN ISSUE:
318 * On a heterogeneous system (big.LITTLE), it will work but the performance
319 * might not be the best one as expected.
320 *
321 * If this function is called on the big core, it will return the function
322 * optimized for the big core.
323 *
324 * If execution is then scheduled to the little core. It will still work (1),
325 * but the function won't be optimized for the little core, thus the performance
326 * won't be as expected.
327 *
328 * References:
329 * - [CPU Feature detection](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/arm64/cpu-feature-registers.rst?h=v5.5)
330 *
331 */
get_micro_arch_id(void)332 static inline uint32_t get_micro_arch_id(void)
333 {
334 uint64_t id=CPU_IMPLEMENTER_RESERVE;
335 #ifndef __APPLE__
336 if ((getauxval(AT_HWCAP) & HWCAP_CPUID)) {
337 /** Here will trap into kernel space */
338 asm("mrs %0, MIDR_EL1 " : "=r" (id));
339 }
340 #endif
341 return id&0xff00fff0;
342 }
343
344
345
346 #endif /* __ASSEMBLY__ */
347 #endif
348