1//===-- xray_trampoline_x86.s -----------------------------------*- ASM -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file is a part of XRay, a dynamic runtime instrumentation system. 10// 11// This implements the X86-specific assembler for the trampolines. 12// 13//===----------------------------------------------------------------------===// 14 15#include "../builtins/assembly.h" 16#include "../sanitizer_common/sanitizer_asm.h" 17 18// XRay trampolines which are not produced by intrinsics are not System V AMD64 19// ABI compliant because they are called with a stack that is always misaligned 20// by 8 bytes with respect to a 16 bytes alignment. This is because they are 21// called immediately after the call to, or immediately before returning from, 22// the function being instrumented. This saves space in the patch point, but 23// misaligns the stack by 8 bytes. 24 25.macro ALIGN_STACK_16B 26#if defined(__APPLE__) 27 subq $$8, %rsp 28#else 29 subq $8, %rsp 30#endif 31 CFI_ADJUST_CFA_OFFSET(8) 32.endm 33 34.macro RESTORE_STACK_ALIGNMENT 35#if defined(__APPLE__) 36 addq $$8, %rsp 37#else 38 addq $8, %rsp 39#endif 40 CFI_ADJUST_CFA_OFFSET(-8) 41.endm 42 43// This macro should lower the stack pointer by an odd multiple of 8. 44.macro SAVE_REGISTERS 45 pushfq 46 CFI_ADJUST_CFA_OFFSET(8) 47 subq $240, %rsp 48 CFI_ADJUST_CFA_OFFSET(240) 49 movq %rbp, 232(%rsp) 50 movupd %xmm0, 216(%rsp) 51 movupd %xmm1, 200(%rsp) 52 movupd %xmm2, 184(%rsp) 53 movupd %xmm3, 168(%rsp) 54 movupd %xmm4, 152(%rsp) 55 movupd %xmm5, 136(%rsp) 56 movupd %xmm6, 120(%rsp) 57 movupd %xmm7, 104(%rsp) 58 movq %rdi, 96(%rsp) 59 movq %rax, 88(%rsp) 60 movq %rdx, 80(%rsp) 61 movq %rsi, 72(%rsp) 62 movq %rcx, 64(%rsp) 63 movq %r8, 56(%rsp) 64 movq %r9, 48(%rsp) 65 movq %r10, 40(%rsp) 66 movq %r11, 32(%rsp) 67 movq %r12, 24(%rsp) 68 movq %r13, 16(%rsp) 69 movq %r14, 8(%rsp) 70 movq %r15, 0(%rsp) 71.endm 72 73.macro RESTORE_REGISTERS 74 movq 232(%rsp), %rbp 75 movupd 216(%rsp), %xmm0 76 movupd 200(%rsp), %xmm1 77 movupd 184(%rsp), %xmm2 78 movupd 168(%rsp), %xmm3 79 movupd 152(%rsp), %xmm4 80 movupd 136(%rsp), %xmm5 81 movupd 120(%rsp) , %xmm6 82 movupd 104(%rsp) , %xmm7 83 movq 96(%rsp), %rdi 84 movq 88(%rsp), %rax 85 movq 80(%rsp), %rdx 86 movq 72(%rsp), %rsi 87 movq 64(%rsp), %rcx 88 movq 56(%rsp), %r8 89 movq 48(%rsp), %r9 90 movq 40(%rsp), %r10 91 movq 32(%rsp), %r11 92 movq 24(%rsp), %r12 93 movq 16(%rsp), %r13 94 movq 8(%rsp), %r14 95 movq 0(%rsp), %r15 96 addq $240, %rsp 97 CFI_ADJUST_CFA_OFFSET(-240) 98 popfq 99 CFI_ADJUST_CFA_OFFSET(-8) 100.endm 101 102 .text 103#if !defined(__APPLE__) 104 .section .text 105 .file "xray_trampoline_x86.S" 106#else 107 .section __TEXT,__text 108#endif 109 110.macro LOAD_HANDLER_ADDR handler 111#if !defined(XRAY_PIC) 112 movq ASM_SYMBOL(\handler)(%rip), %rax 113#else 114 movq ASM_SYMBOL(\handler)@GOTPCREL(%rip), %rax 115 movq (%rax), %rax 116#endif 117.endm 118 119 120//===----------------------------------------------------------------------===// 121 122 .globl ASM_SYMBOL(__xray_FunctionEntry) 123 ASM_HIDDEN(__xray_FunctionEntry) 124 .align 16, 0x90 125 ASM_TYPE_FUNCTION(__xray_FunctionEntry) 126# LLVM-MCA-BEGIN __xray_FunctionEntry 127ASM_SYMBOL(__xray_FunctionEntry): 128 CFI_STARTPROC 129 SAVE_REGISTERS 130 ALIGN_STACK_16B 131 132 // This load has to be atomic, it's concurrent with __xray_patch(). 133 // On x86/amd64, a simple (type-aligned) MOV instruction is enough. 134 LOAD_HANDLER_ADDR _ZN6__xray19XRayPatchedFunctionE 135 testq %rax, %rax 136 je LOCAL_LABEL(tmp0) 137 138 // The patched function prologue puts its xray_instr_map index into %r10d. 139 movl %r10d, %edi 140 xor %esi,%esi 141 callq *%rax 142 143LOCAL_LABEL(tmp0): 144 RESTORE_STACK_ALIGNMENT 145 RESTORE_REGISTERS 146 retq 147# LLVM-MCA-END 148 ASM_SIZE(__xray_FunctionEntry) 149 CFI_ENDPROC 150 151//===----------------------------------------------------------------------===// 152 153 .globl ASM_SYMBOL(__xray_FunctionExit) 154 ASM_HIDDEN(__xray_FunctionExit) 155 .align 16, 0x90 156 ASM_TYPE_FUNCTION(__xray_FunctionExit) 157# LLVM-MCA-BEGIN __xray_FunctionExit 158ASM_SYMBOL(__xray_FunctionExit): 159 CFI_STARTPROC 160 ALIGN_STACK_16B 161 162 // Save the important registers first. Since we're assuming that this 163 // function is only jumped into, we only preserve the registers for 164 // returning. 165 subq $64, %rsp 166 CFI_ADJUST_CFA_OFFSET(64) 167 movq %rbp, 48(%rsp) 168 movupd %xmm0, 32(%rsp) 169 movupd %xmm1, 16(%rsp) 170 movq %rax, 8(%rsp) 171 movq %rdx, 0(%rsp) 172 LOAD_HANDLER_ADDR _ZN6__xray19XRayPatchedFunctionE 173 testq %rax,%rax 174 je LOCAL_LABEL(tmp2) 175 176 movl %r10d, %edi 177 movl $1, %esi 178 callq *%rax 179 180LOCAL_LABEL(tmp2): 181 // Restore the important registers. 182 movq 48(%rsp), %rbp 183 movupd 32(%rsp), %xmm0 184 movupd 16(%rsp), %xmm1 185 movq 8(%rsp), %rax 186 movq 0(%rsp), %rdx 187 addq $64, %rsp 188 CFI_ADJUST_CFA_OFFSET(-64) 189 190 RESTORE_STACK_ALIGNMENT 191 retq 192# LLVM-MCA-END 193 ASM_SIZE(__xray_FunctionExit) 194 CFI_ENDPROC 195 196//===----------------------------------------------------------------------===// 197 198 .globl ASM_SYMBOL(__xray_FunctionTailExit) 199 ASM_HIDDEN(__xray_FunctionTailExit) 200 .align 16, 0x90 201 ASM_TYPE_FUNCTION(__xray_FunctionTailExit) 202# LLVM-MCA-BEGIN __xray_FunctionTailExit 203ASM_SYMBOL(__xray_FunctionTailExit): 204 CFI_STARTPROC 205 SAVE_REGISTERS 206 ALIGN_STACK_16B 207 208 LOAD_HANDLER_ADDR _ZN6__xray19XRayPatchedFunctionE 209 testq %rax,%rax 210 je LOCAL_LABEL(tmp4) 211 212 movl %r10d, %edi 213 movl $2, %esi 214 callq *%rax 215 216LOCAL_LABEL(tmp4): 217 RESTORE_STACK_ALIGNMENT 218 RESTORE_REGISTERS 219 retq 220# LLVM-MCA-END 221 ASM_SIZE(__xray_FunctionTailExit) 222 CFI_ENDPROC 223 224//===----------------------------------------------------------------------===// 225 226 .globl ASM_SYMBOL(__xray_ArgLoggerEntry) 227 ASM_HIDDEN(__xray_ArgLoggerEntry) 228 .align 16, 0x90 229 ASM_TYPE_FUNCTION(__xray_ArgLoggerEntry) 230# LLVM-MCA-BEGIN __xray_ArgLoggerEntry 231ASM_SYMBOL(__xray_ArgLoggerEntry): 232 CFI_STARTPROC 233 SAVE_REGISTERS 234 ALIGN_STACK_16B 235 236 // Again, these function pointer loads must be atomic; MOV is fine. 237 LOAD_HANDLER_ADDR _ZN6__xray13XRayArgLoggerE 238 testq %rax, %rax 239 jne LOCAL_LABEL(arg1entryLog) 240 241 // If [arg1 logging handler] not set, defer to no-arg logging. 242 LOAD_HANDLER_ADDR _ZN6__xray19XRayPatchedFunctionE 243 testq %rax, %rax 244 je LOCAL_LABEL(arg1entryFail) 245 246LOCAL_LABEL(arg1entryLog): 247 248 // First argument will become the third 249 movq %rdi, %rdx 250 251 // XRayEntryType::LOG_ARGS_ENTRY into the second 252 mov $0x3, %esi 253 254 // 32-bit function ID becomes the first 255 movl %r10d, %edi 256 257 callq *%rax 258 259LOCAL_LABEL(arg1entryFail): 260 RESTORE_STACK_ALIGNMENT 261 RESTORE_REGISTERS 262 retq 263# LLVM-MCA-END 264 ASM_SIZE(__xray_ArgLoggerEntry) 265 CFI_ENDPROC 266 267//===----------------------------------------------------------------------===// 268 269// __xray_*Event have default visibility so that they can be referenced by user 270// DSOs that do not link against the runtime. 271 .global ASM_SYMBOL(__xray_CustomEvent) 272 .align 16, 0x90 273 ASM_TYPE_FUNCTION(__xray_CustomEvent) 274# LLVM-MCA-BEGIN __xray_CustomEvent 275ASM_SYMBOL(__xray_CustomEvent): 276 CFI_STARTPROC 277 SAVE_REGISTERS 278 279 // We take two arguments to this trampoline, which should be in rdi and rsi 280 // already. 281 LOAD_HANDLER_ADDR _ZN6__xray22XRayPatchedCustomEventE 282 testq %rax,%rax 283 je LOCAL_LABEL(customEventCleanup) 284 285 callq *%rax 286 287LOCAL_LABEL(customEventCleanup): 288 RESTORE_REGISTERS 289 retq 290# LLVM-MCA-END 291 ASM_SIZE(__xray_CustomEvent) 292 CFI_ENDPROC 293 294//===----------------------------------------------------------------------===// 295 296 .global ASM_SYMBOL(__xray_TypedEvent) 297 .align 16, 0x90 298 ASM_TYPE_FUNCTION(__xray_TypedEvent) 299# LLVM-MCA-BEGIN __xray_TypedEvent 300ASM_SYMBOL(__xray_TypedEvent): 301 CFI_STARTPROC 302 SAVE_REGISTERS 303 304 // We pass three arguments to this trampoline, which should be in rdi, rsi 305 // and rdx without our intervention. 306 LOAD_HANDLER_ADDR _ZN6__xray21XRayPatchedTypedEventE 307 testq %rax,%rax 308 je LOCAL_LABEL(typedEventCleanup) 309 310 callq *%rax 311 312LOCAL_LABEL(typedEventCleanup): 313 RESTORE_REGISTERS 314 retq 315# LLVM-MCA-END 316 ASM_SIZE(__xray_TypedEvent) 317 CFI_ENDPROC 318 319//===----------------------------------------------------------------------===// 320 321NO_EXEC_STACK_DIRECTIVE 322