1 /* $NetBSD: copyin.c,v 1.6 2014/07/24 23:27:25 joerg Exp $ */ 2 3 /*- 4 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects 9 * Agency and which was developed by Matt Thomas of 3am Software Foundry. 10 * 11 * This material is based upon work supported by the Defense Advanced Research 12 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under 13 * Contract No. N66001-09-C-2073. 14 * Approved for Public Release, Distribution Unlimited 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 1. Redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer. 21 * 2. Redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 26 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 #include <sys/cdefs.h> 39 __KERNEL_RCSID(0, "$NetBSD: copyin.c,v 1.6 2014/07/24 23:27:25 joerg Exp $"); 40 41 #include <sys/param.h> 42 #include <sys/lwp.h> 43 44 #include <powerpc/pcb.h> 45 46 #include <powerpc/booke/cpuvar.h> 47 48 static inline uint8_t 49 copyin_byte(const uint8_t * const usaddr8, register_t ds_msr) 50 { 51 register_t msr; 52 uint8_t data; 53 __asm volatile( 54 "mfmsr %[msr]; " /* Save MSR */ 55 "mtmsr %[ds_msr]; sync; isync; " /* DS on */ 56 "lbz %[data],0(%[usaddr8]); " /* fetch user byte */ 57 "mtmsr %[msr]; sync; isync; " /* DS off */ 58 : [msr] "=&r" (msr), [data] "=r" (data) 59 : [ds_msr] "r" (ds_msr), [usaddr8] "b" (usaddr8)); 60 return data; 61 } 62 63 #if 0 64 static inline uint16_t 65 copyin_halfword(const uint16_t * const usaddr16, register_t ds_msr) 66 { 67 register_t msr; 68 uint16_t data; 69 __asm volatile( 70 "mfmsr %[msr]; " /* Save MSR */ 71 "mtmsr %[ds_msr]; sync; isync; " /* DS on */ 72 "lhz %[data],0(%[usaddr16]); " /* fetch user byte */ 73 "mtmsr %[msr]; sync; isync; " /* DS off */ 74 : [msr] "=&r" (msr), [data] "=r" (data) 75 : [ds_msr] "r" (ds_msr), [usaddr16] "b" (usaddr16)); 76 return data; 77 } 78 #endif 79 80 static inline uint32_t 81 copyin_word(const uint32_t * const usaddr32, register_t ds_msr) 82 { 83 register_t msr; 84 uint32_t data; 85 __asm volatile( 86 "mfmsr %[msr]; " /* Save MSR */ 87 "mtmsr %[ds_msr]; sync; isync; " /* DS on */ 88 "lwz %[data],0(%[usaddr32]); " /* load user byte */ 89 "mtmsr %[msr]; sync; isync; " /* DS off */ 90 : [msr] "=&r" (msr), [data] "=r" (data) 91 : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32)); 92 return data; 93 } 94 95 static inline uint32_t 96 copyin_word_bswap(const uint32_t * const usaddr32, register_t ds_msr) 97 { 98 register_t msr; 99 uint32_t data; 100 __asm volatile( 101 "mfmsr %[msr]; " /* Save MSR */ 102 "mtmsr %[ds_msr]; sync; isync; " /* DS on */ 103 "lwbrx %[data],0,%[usaddr32]; " /* load user LE word */ 104 "mtmsr %[msr]; sync; isync; " /* DS off */ 105 : [msr] "=&r" (msr), [data] "=r" (data) 106 : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32)); 107 return data; 108 } 109 110 static inline void 111 copyin_8words(const uint32_t *usaddr32, uint32_t *kdaddr32, register_t ds_msr) 112 { 113 register_t msr; 114 //uint32_t data[8]; 115 __asm volatile( 116 "mfmsr %[msr]" /* Save MSR */ 117 "\n\t" "mtmsr %[ds_msr]; sync; isync" /* DS on */ 118 "\n\t" "lwz %[data0],0(%[usaddr32])" /* fetch user data */ 119 "\n\t" "lwz %[data1],4(%[usaddr32])" /* fetch user data */ 120 "\n\t" "lwz %[data2],8(%[usaddr32])" /* fetch user data */ 121 "\n\t" "lwz %[data3],12(%[usaddr32])" /* fetch user data */ 122 "\n\t" "lwz %[data4],16(%[usaddr32])" /* fetch user data */ 123 "\n\t" "lwz %[data5],20(%[usaddr32])" /* fetch user data */ 124 "\n\t" "lwz %[data6],24(%[usaddr32])" /* fetch user data */ 125 "\n\t" "lwz %[data7],28(%[usaddr32])" /* fetch user data */ 126 "\n\t" "mtmsr %[msr]; sync; isync" /* DS off */ 127 : [msr] "=&r" (msr), 128 [data0] "=&r" (kdaddr32[0]), [data1] "=&r" (kdaddr32[1]), 129 [data2] "=&r" (kdaddr32[2]), [data3] "=&r" (kdaddr32[3]), 130 [data4] "=&r" (kdaddr32[4]), [data5] "=&r" (kdaddr32[5]), 131 [data6] "=&r" (kdaddr32[6]), [data7] "=&r" (kdaddr32[7]) 132 : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32)); 133 } 134 135 static inline void 136 copyin_16words(const uint32_t *usaddr32, uint32_t *kdaddr32, register_t ds_msr) 137 { 138 register_t msr; 139 __asm volatile( 140 "mfmsr %[msr]" /* Save MSR */ 141 "\n\t" "mtmsr %[ds_msr]; sync; isync" /* DS on */ 142 "\n\t" "lwz %[data0],0(%[usaddr32])" /* fetch user data */ 143 "\n\t" "lwz %[data1],4(%[usaddr32])" /* fetch user data */ 144 "\n\t" "lwz %[data2],8(%[usaddr32])" /* fetch user data */ 145 "\n\t" "lwz %[data3],12(%[usaddr32])" /* fetch user data */ 146 "\n\t" "lwz %[data4],16(%[usaddr32])" /* fetch user data */ 147 "\n\t" "lwz %[data5],20(%[usaddr32])" /* fetch user data */ 148 "\n\t" "lwz %[data6],24(%[usaddr32])" /* fetch user data */ 149 "\n\t" "lwz %[data7],28(%[usaddr32])" /* fetch user data */ 150 "\n\t" "lwz %[data8],32(%[usaddr32])" /* fetch user data */ 151 "\n\t" "lwz %[data9],36(%[usaddr32])" /* fetch user data */ 152 "\n\t" "lwz %[data10],40(%[usaddr32])" /* fetch user data */ 153 "\n\t" "lwz %[data11],44(%[usaddr32])" /* fetch user data */ 154 "\n\t" "lwz %[data12],48(%[usaddr32])" /* fetch user data */ 155 "\n\t" "lwz %[data13],52(%[usaddr32])" /* fetch user data */ 156 "\n\t" "lwz %[data14],56(%[usaddr32])" /* fetch user data */ 157 "\n\t" "lwz %[data15],60(%[usaddr32])" /* fetch user data */ 158 "\n\t" "mtmsr %[msr]; sync; isync" /* DS off */ 159 : [msr] "=&r" (msr), 160 [data0] "=&r" (kdaddr32[0]), [data1] "=&r" (kdaddr32[1]), 161 [data2] "=&r" (kdaddr32[2]), [data3] "=&r" (kdaddr32[3]), 162 [data4] "=&r" (kdaddr32[4]), [data5] "=&r" (kdaddr32[5]), 163 [data6] "=&r" (kdaddr32[6]), [data7] "=&r" (kdaddr32[7]), 164 [data8] "=&r" (kdaddr32[8]), [data9] "=&r" (kdaddr32[9]), 165 [data10] "=&r" (kdaddr32[10]), [data11] "=&r" (kdaddr32[11]), 166 [data12] "=&r" (kdaddr32[12]), [data13] "=&r" (kdaddr32[13]), 167 [data14] "=&r" (kdaddr32[14]), [data15] "=&r" (kdaddr32[15]) 168 : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32)); 169 } 170 static inline void 171 copyin_bytes(vaddr_t usaddr, vaddr_t kdaddr, size_t len, register_t ds_msr) 172 { 173 const uint8_t *usaddr8 = (void *)usaddr; 174 uint8_t *kdaddr8 = (void *)kdaddr; 175 while (len-- > 0) { 176 *kdaddr8++ = copyin_byte(usaddr8++, ds_msr); 177 } 178 } 179 180 static inline void 181 copyin_words(vaddr_t usaddr, vaddr_t kdaddr, size_t len, register_t ds_msr) 182 { 183 KASSERT((kdaddr & 3) == 0); 184 KASSERT((usaddr & 3) == 0); 185 const uint32_t *usaddr32 = (void *)usaddr; 186 uint32_t *kdaddr32 = (void *)kdaddr; 187 len >>= 2; 188 while (len >= 16) { 189 copyin_16words(usaddr32, kdaddr32, ds_msr); 190 usaddr32 += 16, kdaddr32 += 16, len -= 16; 191 } 192 KASSERT(len < 16); 193 if (len >= 8) { 194 copyin_8words(usaddr32, kdaddr32, ds_msr); 195 usaddr32 += 8, kdaddr32 += 8, len -= 8; 196 } 197 while (len-- > 0) { 198 *kdaddr32++ = copyin_word(usaddr32++, ds_msr); 199 } 200 } 201 202 uint32_t 203 ufetch_32(const void *vusaddr) 204 { 205 struct pcb * const pcb = lwp_getpcb(curlwp); 206 struct faultbuf env; 207 208 if (setfault(&env) != 0) { 209 pcb->pcb_onfault = NULL; 210 return -1; 211 } 212 213 uint32_t rv = copyin_word(vusaddr, mfmsr() | PSL_DS); 214 215 pcb->pcb_onfault = NULL; 216 217 return rv; 218 } 219 220 int 221 copyin(const void *vusaddr, void *vkdaddr, size_t len) 222 { 223 struct pcb * const pcb = lwp_getpcb(curlwp); 224 struct faultbuf env; 225 vaddr_t usaddr = (vaddr_t) vusaddr; 226 vaddr_t kdaddr = (vaddr_t) vkdaddr; 227 228 if (__predict_false(len == 0)) { 229 return 0; 230 } 231 232 const register_t ds_msr = mfmsr() | PSL_DS; 233 234 int rv = setfault(&env); 235 if (rv != 0) { 236 pcb->pcb_onfault = NULL; 237 return rv; 238 } 239 240 if (__predict_false(len < 4)) { 241 copyin_bytes(usaddr, kdaddr, len, ds_msr); 242 pcb->pcb_onfault = NULL; 243 return 0; 244 } 245 246 const size_t alignment = (usaddr ^ kdaddr) & 3; 247 if (__predict_true(alignment == 0)) { 248 size_t slen; 249 if (__predict_false(kdaddr & 3)) { 250 slen = 4 - (kdaddr & 3); 251 copyin_bytes(usaddr, kdaddr, slen, ds_msr); 252 usaddr += slen, kdaddr += slen, len -= slen; 253 } 254 slen = len & ~3; 255 if (__predict_true(slen >= 4)) { 256 copyin_words(usaddr, kdaddr, slen, ds_msr); 257 usaddr += slen, kdaddr += slen, len -= slen; 258 } 259 } 260 if (len > 0) { 261 copyin_bytes(usaddr, kdaddr, len, ds_msr); 262 } 263 pcb->pcb_onfault = NULL; 264 return 0; 265 } 266 267 int 268 copyinstr(const void *usaddr, void *kdaddr, size_t len, size_t *done) 269 { 270 struct pcb * const pcb = lwp_getpcb(curlwp); 271 struct faultbuf env; 272 273 if (__predict_false(len == 0)) { 274 if (done) 275 *done = 0; 276 return 0; 277 } 278 279 int rv = setfault(&env); 280 if (rv != 0) { 281 pcb->pcb_onfault = NULL; 282 if (done) 283 *done = 0; 284 return rv; 285 } 286 287 const register_t ds_msr = mfmsr() | PSL_DS; 288 const uint32_t *usaddr32 = (const void *)((uintptr_t)usaddr & ~3); 289 uint8_t *kdaddr8 = kdaddr; 290 size_t copylen, wlen; 291 uint32_t data; 292 size_t uoff = (uintptr_t)usaddr & 3; 293 wlen = 4 - uoff; 294 /* 295 * We need discard any leading bytes if the address was 296 * unaligned. We read the words byteswapped so that the LSB 297 * contains the lowest address byte. 298 */ 299 data = copyin_word_bswap(usaddr32++, ds_msr) >> (8 * uoff); 300 for (copylen = 0; copylen < len; copylen++, wlen--, data >>= 8) { 301 if (wlen == 0) { 302 /* 303 * If we've depleted the data in the word, fetch the 304 * next one. 305 */ 306 data = copyin_word_bswap(usaddr32++, ds_msr); 307 wlen = 4; 308 } 309 *kdaddr8++ = data; 310 if ((uint8_t) data == 0) { 311 copylen++; 312 break; 313 } 314 } 315 316 pcb->pcb_onfault = NULL; 317 if (done) 318 *done = copylen; 319 /* 320 * If the last byte is not NUL (0), then the name is too long. 321 */ 322 return (uint8_t)data ? ENAMETOOLONG : 0; 323 } 324