1/* $NetBSD: dtrace_asm.S,v 1.9 2023/11/03 09:07:56 chs Exp $ */ 2 3/* 4 * CDDL HEADER START 5 * 6 * The contents of this file are subject to the terms of the 7 * Common Development and Distribution License (the "License"). 8 * You may not use this file except in compliance with the License. 9 * 10 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 11 * or http://www.opensolaris.org/os/licensing. 12 * See the License for the specific language governing permissions 13 * and limitations under the License. 14 * 15 * When distributing Covered Code, include this CDDL HEADER in each 16 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 17 * If applicable, add the following below this CDDL HEADER, with the 18 * fields enclosed by brackets "[]" replaced with your own identifying 19 * information: Portions Copyright [yyyy] [name of copyright owner] 20 * 21 * CDDL HEADER END 22 * 23 * Portions Copyright 2008 John Birrell <jb@freebsd.org> 24 * 25 * $FreeBSD: head/sys/cddl/dev/dtrace/amd64/dtrace_asm.S 298171 2016-04-17 23:08:47Z markj $ 26 * 27 */ 28/* 29 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 30 * Use is subject to license terms. 31 */ 32 33#define _ASM 34 35#include <sys/cpuvar_defs.h> 36#include <sys/dtrace.h> 37#include <machine/asm.h> 38#define MEXITCOUNT 39 40#include "assym.h" 41 42#define DTRACE_SMAP_DISABLE \ 43 call dtrace_smap_disable 44#define DTRACE_SMAP_ENABLE \ 45 call dtrace_smap_enable 46 47#define INTR_POP \ 48 MEXITCOUNT; \ 49 movq TF_RDI(%rsp),%rdi; \ 50 movq TF_RSI(%rsp),%rsi; \ 51 movq TF_RDX(%rsp),%rdx; \ 52 movq TF_RCX(%rsp),%rcx; \ 53 movq TF_R8(%rsp),%r8; \ 54 movq TF_R9(%rsp),%r9; \ 55 movq TF_RAX(%rsp),%rax; \ 56 movq TF_RBX(%rsp),%rbx; \ 57 movq TF_RBP(%rsp),%rbp; \ 58 movq TF_R10(%rsp),%r10; \ 59 movq TF_R11(%rsp),%r11; \ 60 movq TF_R12(%rsp),%r12; \ 61 movq TF_R13(%rsp),%r13; \ 62 movq TF_R14(%rsp),%r14; \ 63 movq TF_R15(%rsp),%r15; \ 64 testb $SEL_RPL_MASK,TF_CS(%rsp); \ 65 jz 1f; \ 66 cli; \ 67 swapgs; \ 681: addq $TF_RIP,%rsp; 69 70 71 ENTRY(dtrace_invop_start) 72 73 /* 74 * #BP traps with %rip set to the next address. We need to decrement 75 * the value to indicate the address of the int3 (0xcc) instruction 76 * that we substituted. 77 */ 78 movq TF_RIP(%rsp), %rdi 79 decq %rdi 80 movq %rsp, %rsi 81 movq TF_RAX(%rsp), %rdx 82 call dtrace_invop 83 ALTENTRY(dtrace_invop_callsite) 84 cmpl $DTRACE_INVOP_PUSHL_EBP, %eax 85 je bp_push 86 cmpl $DTRACE_INVOP_LEAVE, %eax 87 je bp_leave 88 cmpl $DTRACE_INVOP_NOP, %eax 89 je bp_nop 90 cmpl $DTRACE_INVOP_RET, %eax 91 je bp_ret 92 93 /* When all else fails handle the trap in the usual way. */ 94 jmpq *dtrace_invop_calltrap_addr 95 96bp_push: 97 /* 98 * We must emulate a "pushq %rbp". To do this, we pull the stack 99 * down 8 bytes, and then store the base pointer. 100 */ 101 INTR_POP 102 subq $16, %rsp /* make room for %rbp */ 103 pushq %rax /* push temp */ 104 movq 24(%rsp), %rax /* load calling RIP */ 105 movq %rax, 8(%rsp) /* store calling RIP */ 106 movq 32(%rsp), %rax /* load calling CS */ 107 movq %rax, 16(%rsp) /* store calling CS */ 108 movq 40(%rsp), %rax /* load calling RFLAGS */ 109 movq %rax, 24(%rsp) /* store calling RFLAGS */ 110 movq 48(%rsp), %rax /* load calling RSP */ 111 subq $8, %rax /* make room for %rbp */ 112 movq %rax, 32(%rsp) /* store calling RSP */ 113 movq 56(%rsp), %rax /* load calling SS */ 114 movq %rax, 40(%rsp) /* store calling SS */ 115 movq 32(%rsp), %rax /* reload calling RSP */ 116 movq %rbp, (%rax) /* store %rbp there */ 117 popq %rax /* pop off temp */ 118 iretq /* return from interrupt */ 119 /*NOTREACHED*/ 120 121bp_leave: 122 /* 123 * We must emulate a "leave", which is the same as a "movq %rbp, %rsp" 124 * followed by a "popq %rbp". This is quite a bit simpler on amd64 125 * than it is on i386 -- we can exploit the fact that the %rsp is 126 * explicitly saved to effect the pop without having to reshuffle 127 * the other data pushed for the trap. 128 */ 129 INTR_POP 130 pushq %rax /* push temp */ 131 movq 8(%rsp), %rax /* load calling RIP */ 132 movq %rax, 8(%rsp) /* store calling RIP */ 133 movq (%rbp), %rax /* get new %rbp */ 134 addq $8, %rbp /* adjust new %rsp */ 135 movq %rbp, 32(%rsp) /* store new %rsp */ 136 movq %rax, %rbp /* set new %rbp */ 137 popq %rax /* pop off temp */ 138 iretq /* return from interrupt */ 139 /*NOTREACHED*/ 140 141bp_nop: 142 /* We must emulate a "nop". */ 143 INTR_POP 144 iretq 145 /*NOTREACHED*/ 146 147bp_ret: 148 INTR_POP 149 pushq %rax /* push temp */ 150 movq 32(%rsp), %rax /* load %rsp */ 151 movq (%rax), %rax /* load calling RIP */ 152 movq %rax, 8(%rsp) /* store calling RIP */ 153 addq $8, 32(%rsp) /* adjust new %rsp */ 154 popq %rax /* pop off temp */ 155 iretq /* return from interrupt */ 156 /*NOTREACHED*/ 157 158 END(dtrace_invop_start) 159 160/* 161void dtrace_invop_init(void) 162*/ 163 ENTRY(dtrace_invop_init) 164 movq $dtrace_invop_start, dtrace_invop_jump_addr(%rip) 165 ret 166 END(dtrace_invop_init) 167 168/* 169void dtrace_invop_uninit(void) 170*/ 171 ENTRY(dtrace_invop_uninit) 172 movq $0, dtrace_invop_jump_addr(%rip) 173 ret 174 END(dtrace_invop_uninit) 175 176/* 177greg_t dtrace_getfp(void) 178*/ 179 ENTRY(dtrace_getfp) 180 movq %rbp, %rax 181 ret 182 END(dtrace_getfp) 183 184/* 185uint32_t 186dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) 187*/ 188 ENTRY(dtrace_cas32) 189 movl %esi, %eax 190 lock 191 cmpxchgl %edx, (%rdi) 192 ret 193 END(dtrace_cas32) 194 195/* 196void * 197dtrace_casptr(void *target, void *cmp, void *new) 198*/ 199 ENTRY(dtrace_casptr) 200 movq %rsi, %rax 201 lock 202 cmpxchgq %rdx, (%rdi) 203 ret 204 END(dtrace_casptr) 205 206/* 207uintptr_t 208dtrace_caller(int aframes) 209*/ 210 ENTRY(dtrace_caller) 211 movq $-1, %rax 212 ret 213 END(dtrace_caller) 214 215/* 216void 217dtrace_copy(uintptr_t src, uintptr_t dest, size_t size) 218*/ 219 ENTRY(dtrace_copy) 220 pushq %rbp 221 movq %rsp, %rbp 222 223 xchgq %rdi, %rsi /* make %rsi source, %rdi dest */ 224 movq %rdx, %rcx /* load count */ 225 DTRACE_SMAP_DISABLE 226 repz /* repeat for count ... */ 227 smovb /* move from %ds:rsi to %ed:rdi */ 228 DTRACE_SMAP_ENABLE 229 leave 230 ret 231 END(dtrace_copy) 232 233/* 234void 235dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, 236 volatile uint16_t *flags) 237*/ 238 ENTRY(dtrace_copystr) 239 pushq %rbp 240 movq %rsp, %rbp 241 DTRACE_SMAP_DISABLE 242 2430: 244 movb (%rdi), %al /* load from source */ 245 movb %al, (%rsi) /* store to destination */ 246 addq $1, %rdi /* increment source pointer */ 247 addq $1, %rsi /* increment destination pointer */ 248 subq $1, %rdx /* decrement remaining count */ 249 cmpb $0, %al 250 je 2f 251 testq $0xfff, %rdx /* test if count is 4k-aligned */ 252 jnz 1f /* if not, continue with copying */ 253 testq $CPU_DTRACE_BADADDR, (%rcx) /* load and test dtrace flags */ 254 jnz 2f 2551: 256 cmpq $0, %rdx 257 jne 0b 2582: 259 DTRACE_SMAP_ENABLE 260 leave 261 ret 262 263 END(dtrace_copystr) 264 265/* 266uintptr_t 267dtrace_fulword(void *addr) 268*/ 269 ENTRY(dtrace_fulword) 270 DTRACE_SMAP_DISABLE 271 movq (%rdi), %rax 272 DTRACE_SMAP_ENABLE 273 ret 274 END(dtrace_fulword) 275 276/* 277uint8_t 278dtrace_fuword8_nocheck(void *addr) 279*/ 280 ENTRY(dtrace_fuword8_nocheck) 281 xorq %rax, %rax 282 DTRACE_SMAP_DISABLE 283 movb (%rdi), %al 284 DTRACE_SMAP_ENABLE 285 ret 286 END(dtrace_fuword8_nocheck) 287 288/* 289uint16_t 290dtrace_fuword16_nocheck(void *addr) 291*/ 292 ENTRY(dtrace_fuword16_nocheck) 293 xorq %rax, %rax 294 DTRACE_SMAP_DISABLE 295 movw (%rdi), %ax 296 DTRACE_SMAP_ENABLE 297 ret 298 END(dtrace_fuword16_nocheck) 299 300/* 301uint32_t 302dtrace_fuword32_nocheck(void *addr) 303*/ 304 ENTRY(dtrace_fuword32_nocheck) 305 xorq %rax, %rax 306 DTRACE_SMAP_DISABLE 307 movl (%rdi), %eax 308 DTRACE_SMAP_ENABLE 309 ret 310 END(dtrace_fuword32_nocheck) 311 312/* 313uint64_t 314dtrace_fuword64_nocheck(void *addr) 315*/ 316 ENTRY(dtrace_fuword64_nocheck) 317 DTRACE_SMAP_DISABLE 318 movq (%rdi), %rax 319 DTRACE_SMAP_ENABLE 320 ret 321 END(dtrace_fuword64_nocheck) 322 323/* 324void 325dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, 326 int fault, int fltoffs, uintptr_t illval) 327*/ 328 ENTRY(dtrace_probe_error) 329 pushq %rbp 330 movq %rsp, %rbp 331 subq $0x8, %rsp 332 movq %r9, (%rsp) 333 movq %r8, %r9 334 movq %rcx, %r8 335 movq %rdx, %rcx 336 movq %rsi, %rdx 337 movq %rdi, %rsi 338 movl dtrace_probeid_error(%rip), %edi 339 call dtrace_probe 340 addq $0x8, %rsp 341 leave 342 ret 343 END(dtrace_probe_error) 344 345/* 346void 347dtrace_membar_producer(void) 348*/ 349 ENTRY(dtrace_membar_producer) 350 rep; ret /* use 2 byte return instruction when branch target */ 351 /* AMD Software Optimization Guide - Section 6.2 */ 352 END(dtrace_membar_producer) 353 354/* 355void 356dtrace_membar_consumer(void) 357*/ 358 ENTRY(dtrace_membar_consumer) 359 rep; ret /* use 2 byte return instruction when branch target */ 360 /* AMD Software Optimization Guide - Section 6.2 */ 361 END(dtrace_membar_consumer) 362 363/* 364dtrace_icookie_t 365dtrace_interrupt_disable(void) 366*/ 367 ENTRY(dtrace_interrupt_disable) 368 pushfq 369 popq %rax 370 cli 371 ret 372 END(dtrace_interrupt_disable) 373 374/* 375void 376dtrace_interrupt_enable(dtrace_icookie_t cookie) 377*/ 378 ENTRY(dtrace_interrupt_enable) 379 pushq %rdi 380 popfq 381 ret 382 END(dtrace_interrupt_enable) 383