1 /* $NetBSD: altivec.c,v 1.31 2017/03/16 16:13:20 chs Exp $ */ 2 3 /* 4 * Copyright (C) 1996 Wolfgang Solfrank. 5 * Copyright (C) 1996 TooLs GmbH. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by TooLs GmbH. 19 * 4. The name of TooLs GmbH may not be used to endorse or promote products 20 * derived from this software without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 25 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 27 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 28 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 29 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 30 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 31 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: altivec.c,v 1.31 2017/03/16 16:13:20 chs Exp $"); 36 37 #include "opt_multiprocessor.h" 38 39 #include <sys/param.h> 40 #include <sys/proc.h> 41 #include <sys/systm.h> 42 #include <sys/atomic.h> 43 44 #include <uvm/uvm_extern.h> /* for vcopypage/vzeropage */ 45 46 #include <powerpc/pcb.h> 47 #include <powerpc/altivec.h> 48 #include <powerpc/spr.h> 49 #include <powerpc/oea/spr.h> 50 #include <powerpc/psl.h> 51 52 static void vec_state_load(lwp_t *, u_int); 53 static void vec_state_save(lwp_t *); 54 static void vec_state_release(lwp_t *); 55 56 const pcu_ops_t vec_ops = { 57 .pcu_id = PCU_VEC, 58 .pcu_state_load = vec_state_load, 59 .pcu_state_save = vec_state_save, 60 .pcu_state_release = vec_state_release, 61 }; 62 63 bool 64 vec_used_p(lwp_t *l) 65 { 66 return pcu_valid_p(&vec_ops, l); 67 } 68 69 void 70 vec_mark_used(lwp_t *l) 71 { 72 return pcu_discard(&vec_ops, l, true); 73 } 74 75 void 76 vec_state_load(lwp_t *l, u_int flags) 77 { 78 struct pcb * const pcb = lwp_getpcb(l); 79 80 if ((flags & PCU_VALID) == 0) { 81 memset(&pcb->pcb_vr, 0, sizeof(pcb->pcb_vr)); 82 vec_mark_used(l); 83 } 84 85 if ((flags & PCU_REENABLE) == 0) { 86 /* 87 * Enable AltiVec temporarily (and disable interrupts). 88 */ 89 const register_t msr = mfmsr(); 90 mtmsr((msr & ~PSL_EE) | PSL_VEC); 91 __asm volatile ("isync"); 92 93 /* 94 * Load the vector unit from vreg which is best done in 95 * assembly. 96 */ 97 vec_load_from_vreg(&pcb->pcb_vr); 98 99 /* 100 * Restore MSR (turn off AltiVec) 101 */ 102 mtmsr(msr); 103 __asm volatile ("isync"); 104 } 105 106 /* 107 * VRSAVE will be restored when trap frame returns 108 */ 109 l->l_md.md_utf->tf_vrsave = pcb->pcb_vr.vrsave; 110 111 /* 112 * Mark vector registers as modified. 113 */ 114 l->l_md.md_flags |= PSL_VEC; 115 l->l_md.md_utf->tf_srr1 |= PSL_VEC; 116 } 117 118 void 119 vec_state_save(lwp_t *l) 120 { 121 struct pcb * const pcb = lwp_getpcb(l); 122 123 /* 124 * Turn on AltiVEC, turn off interrupts. 125 */ 126 const register_t msr = mfmsr(); 127 mtmsr((msr & ~PSL_EE) | PSL_VEC); 128 __asm volatile ("isync"); 129 130 /* 131 * Grab contents of vector unit. 132 */ 133 vec_unload_to_vreg(&pcb->pcb_vr); 134 135 /* 136 * Save VRSAVE 137 */ 138 pcb->pcb_vr.vrsave = l->l_md.md_utf->tf_vrsave; 139 140 /* 141 * Note that we aren't using any CPU resources and stop any 142 * data streams. 143 */ 144 __asm volatile ("dssall; sync"); 145 146 /* 147 * Restore MSR (turn off AltiVec) 148 */ 149 mtmsr(msr); 150 __asm volatile ("isync"); 151 } 152 153 void 154 vec_state_release(lwp_t *l) 155 { 156 __asm volatile("dssall;sync"); 157 l->l_md.md_utf->tf_srr1 &= ~PSL_VEC; 158 l->l_md.md_flags &= ~PSL_VEC; 159 } 160 161 void 162 vec_restore_from_mcontext(struct lwp *l, const mcontext_t *mcp) 163 { 164 struct pcb * const pcb = lwp_getpcb(l); 165 166 KASSERT(l == curlwp); 167 168 /* we don't need to save the state, just drop it */ 169 pcu_discard(&vec_ops, l, true); 170 171 memcpy(pcb->pcb_vr.vreg, &mcp->__vrf.__vrs, sizeof (pcb->pcb_vr.vreg)); 172 pcb->pcb_vr.vscr = mcp->__vrf.__vscr; 173 pcb->pcb_vr.vrsave = mcp->__vrf.__vrsave; 174 l->l_md.md_utf->tf_vrsave = pcb->pcb_vr.vrsave; 175 } 176 177 bool 178 vec_save_to_mcontext(struct lwp *l, mcontext_t *mcp, unsigned int *flagp) 179 { 180 struct pcb * const pcb = lwp_getpcb(l); 181 182 KASSERT(l == curlwp); 183 184 /* Save AltiVec context, if any. */ 185 if (!vec_used_p(l)) 186 return false; 187 188 /* 189 * If we're the AltiVec owner, dump its context to the PCB first. 190 */ 191 pcu_save(&vec_ops, l); 192 193 mcp->__gregs[_REG_MSR] |= PSL_VEC; 194 mcp->__vrf.__vscr = pcb->pcb_vr.vscr; 195 mcp->__vrf.__vrsave = l->l_md.md_utf->tf_vrsave; 196 memcpy(mcp->__vrf.__vrs, pcb->pcb_vr.vreg, sizeof (mcp->__vrf.__vrs)); 197 *flagp |= _UC_POWERPC_VEC; 198 return true; 199 } 200 201 #define ZERO_VEC 19 202 203 void 204 vzeropage(paddr_t pa) 205 { 206 const paddr_t ea = pa + PAGE_SIZE; 207 uint32_t vec[7], *vp = (void *) roundup((uintptr_t) vec, 16); 208 register_t omsr, msr; 209 210 __asm volatile("mfmsr %0" : "=r"(omsr) :); 211 212 /* 213 * Turn on AltiVec, turn off interrupts. 214 */ 215 msr = (omsr & ~PSL_EE) | PSL_VEC; 216 __asm volatile("sync; mtmsr %0; isync" :: "r"(msr)); 217 218 /* 219 * Save the VEC register we are going to use before we disable 220 * relocation. 221 */ 222 __asm("stvx %1,0,%0" :: "r"(vp), "n"(ZERO_VEC)); 223 __asm("vxor %0,%0,%0" :: "n"(ZERO_VEC)); 224 225 /* 226 * Zero the page using a single cache line. 227 */ 228 __asm volatile( 229 " sync ;" 230 " mfmsr %[msr];" 231 " rlwinm %[msr],%[msr],0,28,26;" /* Clear PSL_DR */ 232 " mtmsr %[msr];" /* Turn off DMMU */ 233 " isync;" 234 "1: stvx %[zv], %[pa], %[off0];" 235 " stvxl %[zv], %[pa], %[off16];" 236 " stvx %[zv], %[pa], %[off32];" 237 " stvxl %[zv], %[pa], %[off48];" 238 " addi %[pa], %[pa], 64;" 239 " cmplw %[pa], %[ea];" 240 " blt+ 1b;" 241 " ori %[msr], %[msr], 0x10;" /* Set PSL_DR */ 242 " sync;" 243 " mtmsr %[msr];" /* Turn on DMMU */ 244 " isync;" 245 :: [msr] "r"(msr), [pa] "b"(pa), [ea] "b"(ea), 246 [off0] "r"(0), [off16] "r"(16), [off32] "r"(32), [off48] "r"(48), 247 [zv] "n"(ZERO_VEC)); 248 249 /* 250 * Restore VEC register (now that we can access the stack again). 251 */ 252 __asm("lvx %1,0,%0" :: "r"(vp), "n"(ZERO_VEC)); 253 254 /* 255 * Restore old MSR (AltiVec OFF). 256 */ 257 __asm volatile("sync; mtmsr %0; isync" :: "r"(omsr)); 258 } 259 260 #define LO_VEC 16 261 #define HI_VEC 17 262 263 void 264 vcopypage(paddr_t dst, paddr_t src) 265 { 266 const paddr_t edst = dst + PAGE_SIZE; 267 uint32_t vec[11], *vp = (void *) roundup((uintptr_t) vec, 16); 268 register_t omsr, msr; 269 270 __asm volatile("mfmsr %0" : "=r"(omsr) :); 271 272 /* 273 * Turn on AltiVec, turn off interrupts. 274 */ 275 msr = (omsr & ~PSL_EE) | PSL_VEC; 276 __asm volatile("sync; mtmsr %0; isync" :: "r"(msr)); 277 278 /* 279 * Save the VEC registers we will be using before we disable 280 * relocation. 281 */ 282 __asm("stvx %2,%1,%0" :: "b"(vp), "r"( 0), "n"(LO_VEC)); 283 __asm("stvx %2,%1,%0" :: "b"(vp), "r"(16), "n"(HI_VEC)); 284 285 /* 286 * Copy the page using a single cache line, with DMMU 287 * disabled. On most PPCs, two vector registers occupy one 288 * cache line. 289 */ 290 __asm volatile( 291 " sync ;" 292 " mfmsr %[msr];" 293 " rlwinm %[msr],%[msr],0,28,26;" /* Clear PSL_DR */ 294 " mtmsr %[msr];" /* Turn off DMMU */ 295 " isync;" 296 "1: lvx %[lv], %[src], %[off0];" 297 " stvx %[lv], %[dst], %[off0];" 298 " lvxl %[hv], %[src], %[off16];" 299 " stvxl %[hv], %[dst], %[off16];" 300 " addi %[src], %[src], 32;" 301 " addi %[dst], %[dst], 32;" 302 " cmplw %[dst], %[edst];" 303 " blt+ 1b;" 304 " ori %[msr], %[msr], 0x10;" /* Set PSL_DR */ 305 " sync;" 306 " mtmsr %[msr];" /* Turn on DMMU */ 307 " isync;" 308 :: [msr] "r"(msr), [src] "b"(src), [dst] "b"(dst), 309 [edst] "b"(edst), [off0] "r"(0), [off16] "r"(16), 310 [lv] "n"(LO_VEC), [hv] "n"(HI_VEC)); 311 312 /* 313 * Restore VEC registers (now that we can access the stack again). 314 */ 315 __asm("lvx %2,%1,%0" :: "b"(vp), "r"( 0), "n"(LO_VEC)); 316 __asm("lvx %2,%1,%0" :: "b"(vp), "r"(16), "n"(HI_VEC)); 317 318 /* 319 * Restore old MSR (AltiVec OFF). 320 */ 321 __asm volatile("sync; mtmsr %0; isync" :: "r"(omsr)); 322 } 323