1 /**********************************************************************
2 Copyright(c) 2019 Arm Corporation All rights reserved.
3
4 Redistribution and use in source and binary forms, with or without
5 modification, are permitted provided that the following conditions
6 are met:
7 * Redistributions of source code must retain the above copyright
8 notice, this list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright
10 notice, this list of conditions and the following disclaimer in
11 the documentation and/or other materials provided with the
12 distribution.
13 * Neither the name of Arm Corporation nor the names of its
14 contributors may be used to endorse or promote products derived
15 from this software without specific prior written permission.
16
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 **********************************************************************/
29 #ifndef __AARCH64_MULTIBINARY_H__
30 #define __AARCH64_MULTIBINARY_H__
31 #ifndef __aarch64__
32 #error "This file is for aarch64 only"
33 #endif
34 #include <asm/hwcap.h>
35 #ifdef __ASSEMBLY__
36 /**
37 * # mbin_interface : the wrapper layer for isal-l api
38 *
39 * ## references:
40 * * https://sourceware.org/git/gitweb.cgi?p=glibc.git;a=blob;f=sysdeps/aarch64/dl-trampoline.S
41 * * http://infocenter.arm.com/help/topic/com.arm.doc.ihi0055b/IHI0055B_aapcs64.pdf
42 * * https://static.docs.arm.com/ihi0057/b/IHI0057B_aadwarf64.pdf?_ga=2.80574487.1870739014.1564969896-1634778941.1548729310
43 *
44 * ## Usage:
45 * 1. Define dispather function
46 * 2. name must be \name\()_dispatcher
47 * 3. Prototype should be *"void * \name\()_dispatcher"*
48 * 4. The dispather should return the right function pointer , revision and a string information .
49 **/
50 .macro mbin_interface name:req
51 .extern \name\()_dispatcher
52 .section .data
53 .balign 8
54 .global \name\()_dispatcher_info
55 .type \name\()_dispatcher_info,%object
56
57 \name\()_dispatcher_info:
58 .quad \name\()_mbinit //func_entry
59
60 .size \name\()_dispatcher_info,. - \name\()_dispatcher_info
61
62 .balign 8
63 .text
64 \name\()_mbinit:
65 //save lp fp, sub sp
66 .cfi_startproc
67 stp x29, x30, [sp, -224]!
68
69 //add cfi directive to avoid GDB bt cmds error
70 //set cfi(Call Frame Information)
71 .cfi_def_cfa_offset 224
72 .cfi_offset 29, -224
73 .cfi_offset 30, -216
74
75 //save parameter/result/indirect result registers
76 stp x8, x9, [sp, 16]
77 .cfi_offset 8, -208
78 .cfi_offset 9, -200
79 stp x0, x1, [sp, 32]
80 .cfi_offset 0, -192
81 .cfi_offset 1, -184
82 stp x2, x3, [sp, 48]
83 .cfi_offset 2, -176
84 .cfi_offset 3, -168
85 stp x4, x5, [sp, 64]
86 .cfi_offset 4, -160
87 .cfi_offset 5, -152
88 stp x6, x7, [sp, 80]
89 .cfi_offset 6, -144
90 .cfi_offset 7, -136
91 stp q0, q1, [sp, 96]
92 .cfi_offset 64, -128
93 .cfi_offset 65, -112
94 stp q2, q3, [sp, 128]
95 .cfi_offset 66, -96
96 .cfi_offset 67, -80
97 stp q4, q5, [sp, 160]
98 .cfi_offset 68, -64
99 .cfi_offset 69, -48
100 stp q6, q7, [sp, 192]
101 .cfi_offset 70, -32
102 .cfi_offset 71, -16
103
104 /**
105 * The dispatcher functions have the following prototype:
106 * void * function_dispatcher(void)
107 * As the dispatcher is returning a struct, by the AAPCS,
108 */
109
110
111 bl \name\()_dispatcher
112 //restore temp/indirect result registers
113 ldp x8, x9, [sp, 16]
114 .cfi_restore 8
115 .cfi_restore 9
116
117 // save function entry
118 str x0, [x9]
119
120 //restore parameter/result registers
121 ldp x0, x1, [sp, 32]
122 .cfi_restore 0
123 .cfi_restore 1
124 ldp x2, x3, [sp, 48]
125 .cfi_restore 2
126 .cfi_restore 3
127 ldp x4, x5, [sp, 64]
128 .cfi_restore 4
129 .cfi_restore 5
130 ldp x6, x7, [sp, 80]
131 .cfi_restore 6
132 .cfi_restore 7
133 ldp q0, q1, [sp, 96]
134 .cfi_restore 64
135 .cfi_restore 65
136 ldp q2, q3, [sp, 128]
137 .cfi_restore 66
138 .cfi_restore 67
139 ldp q4, q5, [sp, 160]
140 .cfi_restore 68
141 .cfi_restore 69
142 ldp q6, q7, [sp, 192]
143 .cfi_restore 70
144 .cfi_restore 71
145 //save lp fp and sp
146 ldp x29, x30, [sp], 224
147 //restore cfi setting
148 .cfi_restore 30
149 .cfi_restore 29
150 .cfi_def_cfa_offset 0
151 .cfi_endproc
152
153 .global \name
154 .type \name,%function
155 .align 2
156 \name\():
157 adrp x9, :got:\name\()_dispatcher_info
158 ldr x9, [x9, #:got_lo12:\name\()_dispatcher_info]
159 ldr x10,[x9]
160 br x10
161 .size \name,. - \name
162
163 .endm
164
165 /**
166 * mbin_interface_base is used for the interfaces which have only
167 * noarch implementation
168 */
169 .macro mbin_interface_base name:req, base:req
170 .extern \base
171 .section .data
172 .balign 8
173 .global \name\()_dispatcher_info
174 .type \name\()_dispatcher_info,%object
175
176 \name\()_dispatcher_info:
177 .quad \base //func_entry
178 .size \name\()_dispatcher_info,. - \name\()_dispatcher_info
179
180 .balign 8
181 .text
182 .global \name
183 .type \name,%function
184 .align 2
185 \name\():
186 adrp x9, :got:\name\()_dispatcher_info
187 ldr x9, [x9, #:got_lo12:\name\()_dispatcher_info]
188 ldr x10,[x9]
189 br x10
190 .size \name,. - \name
191
192 .endm
193
194 #else /* __ASSEMBLY__ */
195 #include <sys/auxv.h>
196
197
198
199 #define DEFINE_INTERFACE_DISPATCHER(name) \
200 void * name##_dispatcher(void)
201
202 #define PROVIDER_BASIC(name) \
203 PROVIDER_INFO(name##_base)
204
205 #define DO_DIGNOSTIC(x) _Pragma GCC diagnostic ignored "-W"#x
206 #define DO_PRAGMA(x) _Pragma (#x)
207 #define DIGNOSTIC_IGNORE(x) DO_PRAGMA(GCC diagnostic ignored #x)
208 #define DIGNOSTIC_PUSH() DO_PRAGMA(GCC diagnostic push)
209 #define DIGNOSTIC_POP() DO_PRAGMA(GCC diagnostic pop)
210
211
212 #define PROVIDER_INFO(_func_entry) \
213 ({ DIGNOSTIC_PUSH() \
214 DIGNOSTIC_IGNORE(-Wnested-externs) \
215 extern void _func_entry(void); \
216 DIGNOSTIC_POP() \
217 _func_entry; \
218 })
219
220 /**
221 * Micro-Architector definitions
222 * Reference: https://developer.arm.com/docs/ddi0595/f/aarch64-system-registers/midr_el1
223 */
224
225 #define CPU_IMPLEMENTER_RESERVE 0x00
226 #define CPU_IMPLEMENTER_ARM 0x41
227
228
229 #define CPU_PART_CORTEX_A57 0xD07
230 #define CPU_PART_CORTEX_A72 0xD08
231 #define CPU_PART_NEOVERSE_N1 0xD0C
232
233 #define MICRO_ARCH_ID(imp,part) \
234 (((CPU_IMPLEMENTER_##imp&0xff)<<24)|((CPU_PART_##part&0xfff)<<4))
235
236 #ifndef HWCAP_CPUID
237 #define HWCAP_CPUID (1<<11)
238 #endif
239
240 /**
241 * @brief get_micro_arch_id
242 * read micro-architector register instruction if possible.This function
243 * provides microarchitecture information and make microarchitecture optimization
244 * possible. It will trap into kernel due to mrs instruction. So it should
245 * be called only in dispatcher, that will be called only once in program
246 * lifecycle. And HWCAP must be match,That will make sure there are no
247 * illegal instruction errors.
248 *
249 * NOTICE:
250 * - HWCAP_CPUID should be available. Otherwise it returns zero
251 * - It MUST be called inside dispather.
252 * - It MUST meet the HWCAP requirements
253 *
254 * Example:
255 * DEFINE_INTERFACE_DISPATCHER(crc32_iscsi)
256 * {
257 * unsigned long auxval = getauxval(AT_HWCAP);
258 * // MUST do the judgement is MUST.
259 * if ((HWCAP_CRC32 | HWCAP_PMULL) == (auxval & (HWCAP_CRC32 | HWCAP_PMULL))) {
260 * switch (get_micro_arch_id()) {
261 * case MICRO_ARCH_ID(ARM, CORTEX_A57):
262 * return PROVIDER_INFO(crc32_pmull_crc_for_a57);
263 * case MICRO_ARCH_ID(ARM, CORTEX_A72):
264 * return PROVIDER_INFO(crc32_pmull_crc_for_a72);
265 * case MICRO_ARCH_ID(ARM, NEOVERSE_N1):
266 * return PROVIDER_INFO(crc32_pmull_crc_for_n1);
267 * case default:
268 * return PROVIDER_INFO(crc32_pmull_crc_for_others);
269 * }
270 * }
271 * return PROVIDER_BASIC(crc32_iscsi);
272 * }
273 * KNOWN ISSUE:
274 * On a heterogeneous system (big.LITTLE), it will work but the performance
275 * might not be the best one as expected.
276 *
277 * If this function is called on the big core, it will return the function
278 * optimized for the big core.
279 *
280 * If execution is then scheduled to the little core. It will still work (1),
281 * but the function won't be optimized for the little core, thus the performance
282 * won't be as expected.
283 *
284 * References:
285 * - [CPU Feature detection](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/arm64/cpu-feature-registers.rst?h=v5.5)
286 *
287 */
get_micro_arch_id(void)288 static inline uint32_t get_micro_arch_id(void)
289 {
290 uint32_t id=CPU_IMPLEMENTER_RESERVE;
291 if ((getauxval(AT_HWCAP) & HWCAP_CPUID)) {
292
293 asm("mrs %0, MIDR_EL1 " : "=r" (id));
294 }
295 return id&0xff00fff0;
296 }
297
298
299
300 #endif /* __ASSEMBLY__ */
301 #endif
302