1// The content of this file is x86_64-only: 2#if defined(__x86_64__) 3 4#include "sanitizer_common/sanitizer_asm.h" 5 6#if !defined(__APPLE__) 7.section .text 8#else 9.section __TEXT,__text 10#endif 11 12ASM_HIDDEN(__tsan_trace_switch) 13.globl ASM_SYMBOL(__tsan_trace_switch_thunk) 14ASM_SYMBOL(__tsan_trace_switch_thunk): 15 CFI_STARTPROC 16 _CET_ENDBR 17 # Save scratch registers. 18 push %rax 19 CFI_ADJUST_CFA_OFFSET(8) 20 CFI_REL_OFFSET(%rax, 0) 21 push %rcx 22 CFI_ADJUST_CFA_OFFSET(8) 23 CFI_REL_OFFSET(%rcx, 0) 24 push %rdx 25 CFI_ADJUST_CFA_OFFSET(8) 26 CFI_REL_OFFSET(%rdx, 0) 27 push %rsi 28 CFI_ADJUST_CFA_OFFSET(8) 29 CFI_REL_OFFSET(%rsi, 0) 30 push %rdi 31 CFI_ADJUST_CFA_OFFSET(8) 32 CFI_REL_OFFSET(%rdi, 0) 33 push %r8 34 CFI_ADJUST_CFA_OFFSET(8) 35 CFI_REL_OFFSET(%r8, 0) 36 push %r9 37 CFI_ADJUST_CFA_OFFSET(8) 38 CFI_REL_OFFSET(%r9, 0) 39 push %r10 40 CFI_ADJUST_CFA_OFFSET(8) 41 CFI_REL_OFFSET(%r10, 0) 42 push %r11 43 CFI_ADJUST_CFA_OFFSET(8) 44 CFI_REL_OFFSET(%r11, 0) 45 # All XMM registers are caller-saved. 46 sub $0x100, %rsp 47 CFI_ADJUST_CFA_OFFSET(0x100) 48 movdqu %xmm0, 0x0(%rsp) 49 movdqu %xmm1, 0x10(%rsp) 50 movdqu %xmm2, 0x20(%rsp) 51 movdqu %xmm3, 0x30(%rsp) 52 movdqu %xmm4, 0x40(%rsp) 53 movdqu %xmm5, 0x50(%rsp) 54 movdqu %xmm6, 0x60(%rsp) 55 movdqu %xmm7, 0x70(%rsp) 56 movdqu %xmm8, 0x80(%rsp) 57 movdqu %xmm9, 0x90(%rsp) 58 movdqu %xmm10, 0xa0(%rsp) 59 movdqu %xmm11, 0xb0(%rsp) 60 movdqu %xmm12, 0xc0(%rsp) 61 movdqu %xmm13, 0xd0(%rsp) 62 movdqu %xmm14, 0xe0(%rsp) 63 movdqu %xmm15, 0xf0(%rsp) 64 # Align stack frame. 65 push %rbx # non-scratch 66 CFI_ADJUST_CFA_OFFSET(8) 67 CFI_REL_OFFSET(%rbx, 0) 68 mov %rsp, %rbx # save current rsp 69 CFI_DEF_CFA_REGISTER(%rbx) 70 shr $4, %rsp # clear 4 lsb, align to 16 71 shl $4, %rsp 72 73 call ASM_SYMBOL(__tsan_trace_switch) 74 75 # Unalign stack frame back. 76 mov %rbx, %rsp # restore the original rsp 77 CFI_DEF_CFA_REGISTER(%rsp) 78 pop %rbx 79 CFI_ADJUST_CFA_OFFSET(-8) 80 # Restore scratch registers. 81 movdqu 0x0(%rsp), %xmm0 82 movdqu 0x10(%rsp), %xmm1 83 movdqu 0x20(%rsp), %xmm2 84 movdqu 0x30(%rsp), %xmm3 85 movdqu 0x40(%rsp), %xmm4 86 movdqu 0x50(%rsp), %xmm5 87 movdqu 0x60(%rsp), %xmm6 88 movdqu 0x70(%rsp), %xmm7 89 movdqu 0x80(%rsp), %xmm8 90 movdqu 0x90(%rsp), %xmm9 91 movdqu 0xa0(%rsp), %xmm10 92 movdqu 0xb0(%rsp), %xmm11 93 movdqu 0xc0(%rsp), %xmm12 94 movdqu 0xd0(%rsp), %xmm13 95 movdqu 0xe0(%rsp), %xmm14 96 movdqu 0xf0(%rsp), %xmm15 97 add $0x100, %rsp 98 CFI_ADJUST_CFA_OFFSET(-0x100) 99 pop %r11 100 CFI_ADJUST_CFA_OFFSET(-8) 101 pop %r10 102 CFI_ADJUST_CFA_OFFSET(-8) 103 pop %r9 104 CFI_ADJUST_CFA_OFFSET(-8) 105 pop %r8 106 CFI_ADJUST_CFA_OFFSET(-8) 107 pop %rdi 108 CFI_ADJUST_CFA_OFFSET(-8) 109 pop %rsi 110 CFI_ADJUST_CFA_OFFSET(-8) 111 pop %rdx 112 CFI_ADJUST_CFA_OFFSET(-8) 113 pop %rcx 114 CFI_ADJUST_CFA_OFFSET(-8) 115 pop %rax 116 CFI_ADJUST_CFA_OFFSET(-8) 117 CFI_RESTORE(%rax) 118 CFI_RESTORE(%rbx) 119 CFI_RESTORE(%rcx) 120 CFI_RESTORE(%rdx) 121 CFI_RESTORE(%rsi) 122 CFI_RESTORE(%rdi) 123 CFI_RESTORE(%r8) 124 CFI_RESTORE(%r9) 125 CFI_RESTORE(%r10) 126 CFI_RESTORE(%r11) 127 ret 128 CFI_ENDPROC 129 130ASM_HIDDEN(__tsan_report_race) 131.globl ASM_SYMBOL(__tsan_report_race_thunk) 132ASM_SYMBOL(__tsan_report_race_thunk): 133 CFI_STARTPROC 134 _CET_ENDBR 135 # Save scratch registers. 136 push %rax 137 CFI_ADJUST_CFA_OFFSET(8) 138 CFI_REL_OFFSET(%rax, 0) 139 push %rcx 140 CFI_ADJUST_CFA_OFFSET(8) 141 CFI_REL_OFFSET(%rcx, 0) 142 push %rdx 143 CFI_ADJUST_CFA_OFFSET(8) 144 CFI_REL_OFFSET(%rdx, 0) 145 push %rsi 146 CFI_ADJUST_CFA_OFFSET(8) 147 CFI_REL_OFFSET(%rsi, 0) 148 push %rdi 149 CFI_ADJUST_CFA_OFFSET(8) 150 CFI_REL_OFFSET(%rdi, 0) 151 push %r8 152 CFI_ADJUST_CFA_OFFSET(8) 153 CFI_REL_OFFSET(%r8, 0) 154 push %r9 155 CFI_ADJUST_CFA_OFFSET(8) 156 CFI_REL_OFFSET(%r9, 0) 157 push %r10 158 CFI_ADJUST_CFA_OFFSET(8) 159 CFI_REL_OFFSET(%r10, 0) 160 push %r11 161 CFI_ADJUST_CFA_OFFSET(8) 162 CFI_REL_OFFSET(%r11, 0) 163 # All XMM registers are caller-saved. 164 sub $0x100, %rsp 165 CFI_ADJUST_CFA_OFFSET(0x100) 166 movdqu %xmm0, 0x0(%rsp) 167 movdqu %xmm1, 0x10(%rsp) 168 movdqu %xmm2, 0x20(%rsp) 169 movdqu %xmm3, 0x30(%rsp) 170 movdqu %xmm4, 0x40(%rsp) 171 movdqu %xmm5, 0x50(%rsp) 172 movdqu %xmm6, 0x60(%rsp) 173 movdqu %xmm7, 0x70(%rsp) 174 movdqu %xmm8, 0x80(%rsp) 175 movdqu %xmm9, 0x90(%rsp) 176 movdqu %xmm10, 0xa0(%rsp) 177 movdqu %xmm11, 0xb0(%rsp) 178 movdqu %xmm12, 0xc0(%rsp) 179 movdqu %xmm13, 0xd0(%rsp) 180 movdqu %xmm14, 0xe0(%rsp) 181 movdqu %xmm15, 0xf0(%rsp) 182 # Align stack frame. 183 push %rbx # non-scratch 184 CFI_ADJUST_CFA_OFFSET(8) 185 CFI_REL_OFFSET(%rbx, 0) 186 mov %rsp, %rbx # save current rsp 187 CFI_DEF_CFA_REGISTER(%rbx) 188 shr $4, %rsp # clear 4 lsb, align to 16 189 shl $4, %rsp 190 191 call ASM_SYMBOL(__tsan_report_race) 192 193 # Unalign stack frame back. 194 mov %rbx, %rsp # restore the original rsp 195 CFI_DEF_CFA_REGISTER(%rsp) 196 pop %rbx 197 CFI_ADJUST_CFA_OFFSET(-8) 198 # Restore scratch registers. 199 movdqu 0x0(%rsp), %xmm0 200 movdqu 0x10(%rsp), %xmm1 201 movdqu 0x20(%rsp), %xmm2 202 movdqu 0x30(%rsp), %xmm3 203 movdqu 0x40(%rsp), %xmm4 204 movdqu 0x50(%rsp), %xmm5 205 movdqu 0x60(%rsp), %xmm6 206 movdqu 0x70(%rsp), %xmm7 207 movdqu 0x80(%rsp), %xmm8 208 movdqu 0x90(%rsp), %xmm9 209 movdqu 0xa0(%rsp), %xmm10 210 movdqu 0xb0(%rsp), %xmm11 211 movdqu 0xc0(%rsp), %xmm12 212 movdqu 0xd0(%rsp), %xmm13 213 movdqu 0xe0(%rsp), %xmm14 214 movdqu 0xf0(%rsp), %xmm15 215 add $0x100, %rsp 216 CFI_ADJUST_CFA_OFFSET(-0x100) 217 pop %r11 218 CFI_ADJUST_CFA_OFFSET(-8) 219 pop %r10 220 CFI_ADJUST_CFA_OFFSET(-8) 221 pop %r9 222 CFI_ADJUST_CFA_OFFSET(-8) 223 pop %r8 224 CFI_ADJUST_CFA_OFFSET(-8) 225 pop %rdi 226 CFI_ADJUST_CFA_OFFSET(-8) 227 pop %rsi 228 CFI_ADJUST_CFA_OFFSET(-8) 229 pop %rdx 230 CFI_ADJUST_CFA_OFFSET(-8) 231 pop %rcx 232 CFI_ADJUST_CFA_OFFSET(-8) 233 pop %rax 234 CFI_ADJUST_CFA_OFFSET(-8) 235 CFI_RESTORE(%rax) 236 CFI_RESTORE(%rbx) 237 CFI_RESTORE(%rcx) 238 CFI_RESTORE(%rdx) 239 CFI_RESTORE(%rsi) 240 CFI_RESTORE(%rdi) 241 CFI_RESTORE(%r8) 242 CFI_RESTORE(%r9) 243 CFI_RESTORE(%r10) 244 CFI_RESTORE(%r11) 245 ret 246 CFI_ENDPROC 247 248ASM_HIDDEN(__tsan_setjmp) 249#if defined(__NetBSD__) 250.comm _ZN14__interception15real___setjmp14E,8,8 251#elif !defined(__APPLE__) 252.comm _ZN14__interception11real_setjmpE,8,8 253#endif 254#if defined(__NetBSD__) 255.globl ASM_SYMBOL_INTERCEPTOR(__setjmp14) 256ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__setjmp14)) 257ASM_SYMBOL_INTERCEPTOR(__setjmp14): 258#else 259.globl ASM_SYMBOL_INTERCEPTOR(setjmp) 260ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp)) 261ASM_SYMBOL_INTERCEPTOR(setjmp): 262#endif 263 CFI_STARTPROC 264 _CET_ENDBR 265 // save env parameter 266 push %rdi 267 CFI_ADJUST_CFA_OFFSET(8) 268 CFI_REL_OFFSET(%rdi, 0) 269 // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` 270#if defined(__FreeBSD__) || defined(__NetBSD__) 271 lea 8(%rsp), %rdi 272#elif defined(__linux__) || defined(__APPLE__) 273 lea 16(%rsp), %rdi 274#else 275# error "Unknown platform" 276#endif 277 // call tsan interceptor 278 call ASM_SYMBOL(__tsan_setjmp) 279 // restore env parameter 280 pop %rdi 281 CFI_ADJUST_CFA_OFFSET(-8) 282 CFI_RESTORE(%rdi) 283 // tail jump to libc setjmp 284 movl $0, %eax 285#if defined(__NetBSD__) 286 movq _ZN14__interception15real___setjmp14E@GOTPCREL(%rip), %rdx 287 jmp *(%rdx) 288#elif !defined(__APPLE__) 289 movq _ZN14__interception11real_setjmpE@GOTPCREL(%rip), %rdx 290 jmp *(%rdx) 291#else 292 jmp ASM_SYMBOL(setjmp) 293#endif 294 CFI_ENDPROC 295#if defined(__NetBSD__) 296ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__setjmp14)) 297#else 298ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp)) 299#endif 300 301.comm _ZN14__interception12real__setjmpE,8,8 302.globl ASM_SYMBOL_INTERCEPTOR(_setjmp) 303ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp)) 304ASM_SYMBOL_INTERCEPTOR(_setjmp): 305 CFI_STARTPROC 306 _CET_ENDBR 307 // save env parameter 308 push %rdi 309 CFI_ADJUST_CFA_OFFSET(8) 310 CFI_REL_OFFSET(%rdi, 0) 311 // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` 312#if defined(__FreeBSD__) || defined(__NetBSD__) 313 lea 8(%rsp), %rdi 314#elif defined(__linux__) || defined(__APPLE__) 315 lea 16(%rsp), %rdi 316#else 317# error "Unknown platform" 318#endif 319 // call tsan interceptor 320 call ASM_SYMBOL(__tsan_setjmp) 321 // restore env parameter 322 pop %rdi 323 CFI_ADJUST_CFA_OFFSET(-8) 324 CFI_RESTORE(%rdi) 325 // tail jump to libc setjmp 326 movl $0, %eax 327#if !defined(__APPLE__) 328 movq _ZN14__interception12real__setjmpE@GOTPCREL(%rip), %rdx 329 jmp *(%rdx) 330#else 331 jmp ASM_SYMBOL(_setjmp) 332#endif 333 CFI_ENDPROC 334ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp)) 335 336#if defined(__NetBSD__) 337.comm _ZN14__interception18real___sigsetjmp14E,8,8 338.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14) 339ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)) 340ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14): 341#else 342.comm _ZN14__interception14real_sigsetjmpE,8,8 343.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp) 344ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) 345ASM_SYMBOL_INTERCEPTOR(sigsetjmp): 346#endif 347 CFI_STARTPROC 348 _CET_ENDBR 349 // save env parameter 350 push %rdi 351 CFI_ADJUST_CFA_OFFSET(8) 352 CFI_REL_OFFSET(%rdi, 0) 353 // save savesigs parameter 354 push %rsi 355 CFI_ADJUST_CFA_OFFSET(8) 356 CFI_REL_OFFSET(%rsi, 0) 357 // align stack frame 358 sub $8, %rsp 359 CFI_ADJUST_CFA_OFFSET(8) 360 // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` 361#if defined(__FreeBSD__) || defined(__NetBSD__) 362 lea 24(%rsp), %rdi 363#elif defined(__linux__) || defined(__APPLE__) 364 lea 32(%rsp), %rdi 365#else 366# error "Unknown platform" 367#endif 368 // call tsan interceptor 369 call ASM_SYMBOL(__tsan_setjmp) 370 // unalign stack frame 371 add $8, %rsp 372 CFI_ADJUST_CFA_OFFSET(-8) 373 // restore savesigs parameter 374 pop %rsi 375 CFI_ADJUST_CFA_OFFSET(-8) 376 CFI_RESTORE(%rsi) 377 // restore env parameter 378 pop %rdi 379 CFI_ADJUST_CFA_OFFSET(-8) 380 CFI_RESTORE(%rdi) 381 // tail jump to libc sigsetjmp 382 movl $0, %eax 383#if defined(__NetBSD__) 384 movq _ZN14__interception18real___sigsetjmp14E@GOTPCREL(%rip), %rdx 385 jmp *(%rdx) 386#elif !defined(__APPLE__) 387 movq _ZN14__interception14real_sigsetjmpE@GOTPCREL(%rip), %rdx 388 jmp *(%rdx) 389#else 390 jmp ASM_SYMBOL(sigsetjmp) 391#endif 392 CFI_ENDPROC 393#if defined(__NetBSD__) 394ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)) 395#else 396ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) 397#endif 398 399#if !defined(__APPLE__) && !defined(__NetBSD__) 400.comm _ZN14__interception16real___sigsetjmpE,8,8 401.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp) 402ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) 403ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): 404 CFI_STARTPROC 405 _CET_ENDBR 406 // save env parameter 407 push %rdi 408 CFI_ADJUST_CFA_OFFSET(8) 409 CFI_REL_OFFSET(%rdi, 0) 410 // save savesigs parameter 411 push %rsi 412 CFI_ADJUST_CFA_OFFSET(8) 413 CFI_REL_OFFSET(%rsi, 0) 414 // align stack frame 415 sub $8, %rsp 416 CFI_ADJUST_CFA_OFFSET(8) 417 // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` 418#if defined(__FreeBSD__) 419 lea 24(%rsp), %rdi 420#else 421 lea 32(%rsp), %rdi 422#endif 423 // call tsan interceptor 424 call ASM_SYMBOL(__tsan_setjmp) 425 // unalign stack frame 426 add $8, %rsp 427 CFI_ADJUST_CFA_OFFSET(-8) 428 // restore savesigs parameter 429 pop %rsi 430 CFI_ADJUST_CFA_OFFSET(-8) 431 CFI_RESTORE(%rsi) 432 // restore env parameter 433 pop %rdi 434 CFI_ADJUST_CFA_OFFSET(-8) 435 CFI_RESTORE(%rdi) 436 // tail jump to libc sigsetjmp 437 movl $0, %eax 438 movq _ZN14__interception16real___sigsetjmpE@GOTPCREL(%rip), %rdx 439 jmp *(%rdx) 440 CFI_ENDPROC 441ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) 442#endif // !defined(__APPLE__) && !defined(__NetBSD__) 443 444NO_EXEC_STACK_DIRECTIVE 445 446#endif 447