1/* $OpenBSD: copy.S,v 1.10 2023/08/12 06:28:13 miod Exp $ */ 2/* 3 * Copyright (c) 2015 Dale Rahn <drahn@dalerahn.com> 4 * Copyright (c) 2014 Patrick Wildt <patrick@blueri.se> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19#include "assym.h" 20#include <sys/syscall.h> 21#include <machine/asm.h> 22#include <sys/errno.h> 23 24 .text 25 .align 2 26 27/* 28 * x0 = user space address 29 * x1 = kernel space address 30 * x2 = length 31 * 32 * Copies bytes from user space to kernel space 33 */ 34ENTRY(copyin) 35 RETGUARD_SETUP(copy, x15) 36 cbnz x2, 1f 37 mov x0, 0 38 RETGUARD_CHECK(copy, x15) 39 ret 40 411: 42 /* Check whether source+len overflows. */ 43 adds x3, x0, x2 44 b.cs .Laddrfault 45 /* Check that source+len is in userspace. */ 46 tst x3, #(1ULL << 63) 47 b.ne .Laddrfault 48 49 mrs x3, tpidr_el1 // load cpuinfo 50 ldr x3, [x3, #(CI_CURPCB)] 51 ldr x4, [x3, #(PCB_ONFAULT)] 52 adr x5, .Lcopyfault 53 str x5, [x3, #(PCB_ONFAULT)] // set handler 54 55 cmp x2, #16 56 b.lo .Lcopyin1 572: ldtr x6, [x0] 58 ldtr x7, [x0, #8] 59 stp x6, x7, [x1], #16 60 add x0, x0, #16 61 sub x2, x2, #16 62 cmp x2, #16 63 b.hs 2b 64 65.Lcopyin1: 66 cbz x2, .Lcopyin0 673: ldtrb w6, [x0] 68 strb w6, [x1], #1 69 add x0, x0, #1 70 sub x2, x2, #1 71 cbnz x2, 3b 72 73.Lcopyin0: 74 str x4, [x3, #(PCB_ONFAULT)] // clear handler 75 mov x0, xzr 76 RETGUARD_CHECK(copy, x15) 77 ret 78 79.Lcopyfault: 80 str x4, [x3, #(PCB_ONFAULT)] 81.Laddrfault: 82 mov x0, #EFAULT 83 RETGUARD_CHECK(copy, x15) 84 ret 85 86/* 87 * x0 = user space address 88 * x1 = kernel space address 89 * 90 * Atomically copies a 32-bit word from user space to kernel space 91 */ 92ENTRY(copyin32) 93 RETGUARD_SETUP(copy, x15) 94 95 /* Check source alignment. */ 96 tst x0, #0x3 97 b.ne .Laddrfault 98 /* Check that source is in userspace. */ 99 tst x0, #(1ULL << 63) 100 b.ne .Laddrfault 101 102 mrs x3, tpidr_el1 // load cpuinfo 103 ldr x3, [x3, #(CI_CURPCB)] 104 ldr x4, [x3, #(PCB_ONFAULT)] 105 adr x5, .Lcopyfault 106 str x5, [x3, #(PCB_ONFAULT)] // set handler 107 108 ldtr w6, [x0] 109 str w6, [x1] 110 111 str x4, [x3, #(PCB_ONFAULT)] // clear handler 112 mov x0, xzr 113 RETGUARD_CHECK(copy, x15) 114 ret 115 116/* 117 * x0 = kernel space address 118 * x1 = user space address 119 * x2 = length 120 * 121 * Copies bytes from kernel space to user space 122 */ 123 124ENTRY(copyout) 125 RETGUARD_SETUP(copy, x15) 126 cbnz x2, 1f 127 mov x0, 0 128 RETGUARD_CHECK(copy, x15) 129 ret 130 1311: 132 /* Check whether dest+len overflows. */ 133 adds x3, x1, x2 134 b.cs .Laddrfault 135 /* Check that dest+len is in userspace. */ 136 tst x3, #(1ULL << 63) 137 b.ne .Laddrfault 138 139 mrs x3, tpidr_el1 // load cpuinfo 140 ldr x3, [x3, #(CI_CURPCB)] 141 ldr x4, [x3, #(PCB_ONFAULT)] 142 adr x5, .Lcopyfault 143 str x5, [x3, #(PCB_ONFAULT)] // set handler 144 145 cmp x2, #16 146 b.lo .Lcopyout1 1472: ldp x6, x7, [x0], #16 148 sttr x6, [x1] 149 sttr x7, [x1, #8] 150 add x1, x1, #16 151 sub x2, x2, #16 152 cmp x2, #16 153 b.hs 2b 154 155.Lcopyout1: 156 cbz x2, .Lcopyout0 1573: ldrb w6, [x0], #1 158 sttrb w6, [x1] 159 add x1, x1, #1 160 sub x2, x2, #1 161 cbnz x2, 3b 162 163.Lcopyout0: 164 str x4, [x3, #(PCB_ONFAULT)] // restore handler 165 mov x0, xzr 166 RETGUARD_CHECK(copy, x15) 167 ret 168 169/* 170 * x0 = kernel space source address 171 * x1 = kernel space destination address 172 * x2 = length 173 * 174 * Copies bytes from kernel space to kernel space, aborting on page fault 175 */ 176 177ENTRY(kcopy) 178 RETGUARD_SETUP(copy, x15) 179 cbnz x2, 1f 180 mov x0, 0 181 RETGUARD_CHECK(copy, x15) 182 ret 1831: 184 mrs x3, tpidr_el1 // load cpuinfo 185 ldr x3, [x3, #(CI_CURPCB)] 186 ldr x4, [x3, #(PCB_ONFAULT)] 187 adr x5, .Lcopyfault 188 str x5, [x3, #(PCB_ONFAULT)] // set handler 189 190 cmp x2, #16 191 b.lo .Lkcopy8 1922: ldp x6, x7, [x0], #16 193 stp x6, x7, [x1], #16 194 sub x2, x2, #16 195 cmp x2, #16 196 b.hs 2b 197 198.Lkcopy8: 199 tbz x2, #3, .Lkcopy4 200 ldr x6, [x0], #8 201 str x6, [x1], #8 202 sub x2, x2, #8 203 204.Lkcopy4: 205 tbz x2, #2, .Lkcopy1 206 ldr w6, [x0], #4 207 str w6, [x1], #4 208 sub x2, x2, #4 209 210.Lkcopy1: 211 cbz x2, .Lkcopy0 2123: ldrb w6, [x0], #1 213 strb w6, [x1], #1 214 sub x2, x2, #1 215 cbnz x2, 3b 216 217.Lkcopy0: 218 str x4, [x3, #(PCB_ONFAULT)] // restore handler 219 mov x0, xzr 220 RETGUARD_CHECK(copy, x15) 221 ret 222