1 /* $NetBSD: altivec.c,v 1.34 2021/10/30 19:44:56 thorpej Exp $ */ 2 3 /* 4 * Copyright (C) 1996 Wolfgang Solfrank. 5 * Copyright (C) 1996 TooLs GmbH. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by TooLs GmbH. 19 * 4. The name of TooLs GmbH may not be used to endorse or promote products 20 * derived from this software without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 25 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 27 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 28 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 29 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 30 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 31 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: altivec.c,v 1.34 2021/10/30 19:44:56 thorpej Exp $"); 36 37 #include <sys/param.h> 38 #include <sys/proc.h> 39 #include <sys/systm.h> 40 #include <sys/atomic.h> 41 42 #include <uvm/uvm_extern.h> /* for vcopypage/vzeropage */ 43 44 #include <powerpc/pcb.h> 45 #include <powerpc/altivec.h> 46 #include <powerpc/spr.h> 47 #include <powerpc/oea/spr.h> 48 #include <powerpc/psl.h> 49 50 static void vec_state_load(lwp_t *, u_int); 51 static void vec_state_save(lwp_t *); 52 static void vec_state_release(lwp_t *); 53 54 const pcu_ops_t vec_ops = { 55 .pcu_id = PCU_VEC, 56 .pcu_state_load = vec_state_load, 57 .pcu_state_save = vec_state_save, 58 .pcu_state_release = vec_state_release, 59 }; 60 61 bool 62 vec_used_p(lwp_t *l) 63 { 64 return pcu_valid_p(&vec_ops, l); 65 } 66 67 void 68 vec_mark_used(lwp_t *l) 69 { 70 return pcu_discard(&vec_ops, l, true); 71 } 72 73 void 74 vec_state_load(lwp_t *l, u_int flags) 75 { 76 struct pcb * const pcb = lwp_getpcb(l); 77 78 if ((flags & PCU_VALID) == 0) { 79 memset(&pcb->pcb_vr, 0, sizeof(pcb->pcb_vr)); 80 vec_mark_used(l); 81 } 82 83 if ((flags & PCU_REENABLE) == 0) { 84 /* 85 * Enable AltiVec temporarily (and disable interrupts). 86 */ 87 const register_t msr = mfmsr(); 88 mtmsr((msr & ~PSL_EE) | PSL_VEC); 89 __asm volatile ("isync"); 90 91 /* 92 * Load the vector unit from vreg which is best done in 93 * assembly. 94 */ 95 vec_load_from_vreg(&pcb->pcb_vr); 96 97 /* 98 * Restore MSR (turn off AltiVec) 99 */ 100 mtmsr(msr); 101 __asm volatile ("isync"); 102 } 103 104 /* 105 * VRSAVE will be restored when trap frame returns 106 */ 107 l->l_md.md_utf->tf_vrsave = pcb->pcb_vr.vrsave; 108 109 /* 110 * Mark vector registers as modified. 111 */ 112 l->l_md.md_flags |= PSL_VEC; 113 l->l_md.md_utf->tf_srr1 |= PSL_VEC; 114 } 115 116 void 117 vec_state_save(lwp_t *l) 118 { 119 struct pcb * const pcb = lwp_getpcb(l); 120 121 /* 122 * Turn on AltiVEC, turn off interrupts. 123 */ 124 const register_t msr = mfmsr(); 125 mtmsr((msr & ~PSL_EE) | PSL_VEC); 126 __asm volatile ("isync"); 127 128 /* 129 * Grab contents of vector unit. 130 */ 131 vec_unload_to_vreg(&pcb->pcb_vr); 132 133 /* 134 * Save VRSAVE 135 */ 136 pcb->pcb_vr.vrsave = l->l_md.md_utf->tf_vrsave; 137 138 /* 139 * Note that we aren't using any CPU resources and stop any 140 * data streams. 141 */ 142 __asm volatile ("dssall; sync"); 143 144 /* 145 * Restore MSR (turn off AltiVec) 146 */ 147 mtmsr(msr); 148 __asm volatile ("isync"); 149 } 150 151 void 152 vec_state_release(lwp_t *l) 153 { 154 __asm volatile("dssall;sync"); 155 l->l_md.md_utf->tf_srr1 &= ~PSL_VEC; 156 l->l_md.md_flags &= ~PSL_VEC; 157 } 158 159 void 160 vec_restore_from_mcontext(struct lwp *l, const mcontext_t *mcp) 161 { 162 struct pcb * const pcb = lwp_getpcb(l); 163 164 KASSERT(l == curlwp); 165 166 /* we don't need to save the state, just drop it */ 167 pcu_discard(&vec_ops, l, true); 168 169 if (mcp != NULL) { /* XXX see compat_16_sys___sigreturn14() */ 170 memcpy(pcb->pcb_vr.vreg, &mcp->__vrf.__vrs, 171 sizeof (pcb->pcb_vr.vreg)); 172 pcb->pcb_vr.vscr = mcp->__vrf.__vscr; 173 pcb->pcb_vr.vrsave = mcp->__vrf.__vrsave; 174 l->l_md.md_utf->tf_vrsave = pcb->pcb_vr.vrsave; 175 } 176 } 177 178 bool 179 vec_save_to_mcontext(struct lwp *l, mcontext_t *mcp, unsigned int *flagp) 180 { 181 struct pcb * const pcb = lwp_getpcb(l); 182 183 KASSERT(l == curlwp); 184 185 /* Save AltiVec context, if any. */ 186 if (!vec_used_p(l)) 187 return false; 188 189 /* 190 * If we're the AltiVec owner, dump its context to the PCB first. 191 */ 192 pcu_save(&vec_ops, l); 193 194 if (mcp != NULL) { /* XXX see sendsig_sigcontext() */ 195 mcp->__gregs[_REG_MSR] |= PSL_VEC; 196 mcp->__vrf.__vscr = pcb->pcb_vr.vscr; 197 mcp->__vrf.__vrsave = l->l_md.md_utf->tf_vrsave; 198 memcpy(mcp->__vrf.__vrs, pcb->pcb_vr.vreg, 199 sizeof (mcp->__vrf.__vrs)); 200 *flagp |= _UC_POWERPC_VEC; 201 } 202 return true; 203 } 204 205 #define ZERO_VEC 19 206 207 void 208 vzeropage(paddr_t pa) 209 { 210 const paddr_t ea = pa + PAGE_SIZE; 211 uint32_t vec[7], *vp = (void *) roundup((uintptr_t) vec, 16); 212 register_t omsr, msr; 213 214 __asm volatile("mfmsr %0" : "=r"(omsr) :); 215 216 /* 217 * Turn on AltiVec, turn off interrupts. 218 */ 219 msr = (omsr & ~PSL_EE) | PSL_VEC; 220 __asm volatile("sync; mtmsr %0; isync" :: "r"(msr)); 221 222 /* 223 * Save the VEC register we are going to use before we disable 224 * relocation. 225 */ 226 __asm("stvx %1,0,%0" :: "r"(vp), "n"(ZERO_VEC)); 227 __asm("vxor %0,%0,%0" :: "n"(ZERO_VEC)); 228 229 /* 230 * Zero the page using a single cache line. 231 */ 232 __asm volatile( 233 " sync ;" 234 " mfmsr %[msr];" 235 " rlwinm %[msr],%[msr],0,28,26;" /* Clear PSL_DR */ 236 " mtmsr %[msr];" /* Turn off DMMU */ 237 " isync;" 238 "1: stvx %[zv], %[pa], %[off0];" 239 " stvxl %[zv], %[pa], %[off16];" 240 " stvx %[zv], %[pa], %[off32];" 241 " stvxl %[zv], %[pa], %[off48];" 242 " addi %[pa], %[pa], 64;" 243 " cmplw %[pa], %[ea];" 244 " blt+ 1b;" 245 " ori %[msr], %[msr], 0x10;" /* Set PSL_DR */ 246 " sync;" 247 " mtmsr %[msr];" /* Turn on DMMU */ 248 " isync;" 249 :: [msr] "r"(msr), [pa] "b"(pa), [ea] "b"(ea), 250 [off0] "r"(0), [off16] "r"(16), [off32] "r"(32), [off48] "r"(48), 251 [zv] "n"(ZERO_VEC)); 252 253 /* 254 * Restore VEC register (now that we can access the stack again). 255 */ 256 __asm("lvx %1,0,%0" :: "r"(vp), "n"(ZERO_VEC)); 257 258 /* 259 * Restore old MSR (AltiVec OFF). 260 */ 261 __asm volatile("sync; mtmsr %0; isync" :: "r"(omsr)); 262 } 263 264 #define LO_VEC 16 265 #define HI_VEC 17 266 267 void 268 vcopypage(paddr_t dst, paddr_t src) 269 { 270 const paddr_t edst = dst + PAGE_SIZE; 271 uint32_t vec[11], *vp = (void *) roundup((uintptr_t) vec, 16); 272 register_t omsr, msr; 273 274 __asm volatile("mfmsr %0" : "=r"(omsr) :); 275 276 /* 277 * Turn on AltiVec, turn off interrupts. 278 */ 279 msr = (omsr & ~PSL_EE) | PSL_VEC; 280 __asm volatile("sync; mtmsr %0; isync" :: "r"(msr)); 281 282 /* 283 * Save the VEC registers we will be using before we disable 284 * relocation. 285 */ 286 __asm("stvx %2,%1,%0" :: "b"(vp), "r"( 0), "n"(LO_VEC)); 287 __asm("stvx %2,%1,%0" :: "b"(vp), "r"(16), "n"(HI_VEC)); 288 289 /* 290 * Copy the page using a single cache line, with DMMU 291 * disabled. On most PPCs, two vector registers occupy one 292 * cache line. 293 */ 294 __asm volatile( 295 " sync ;" 296 " mfmsr %[msr];" 297 " rlwinm %[msr],%[msr],0,28,26;" /* Clear PSL_DR */ 298 " mtmsr %[msr];" /* Turn off DMMU */ 299 " isync;" 300 "1: lvx %[lv], %[src], %[off0];" 301 " stvx %[lv], %[dst], %[off0];" 302 " lvxl %[hv], %[src], %[off16];" 303 " stvxl %[hv], %[dst], %[off16];" 304 " addi %[src], %[src], 32;" 305 " addi %[dst], %[dst], 32;" 306 " cmplw %[dst], %[edst];" 307 " blt+ 1b;" 308 " ori %[msr], %[msr], 0x10;" /* Set PSL_DR */ 309 " sync;" 310 " mtmsr %[msr];" /* Turn on DMMU */ 311 " isync;" 312 :: [msr] "r"(msr), [src] "b"(src), [dst] "b"(dst), 313 [edst] "b"(edst), [off0] "r"(0), [off16] "r"(16), 314 [lv] "n"(LO_VEC), [hv] "n"(HI_VEC)); 315 316 /* 317 * Restore VEC registers (now that we can access the stack again). 318 */ 319 __asm("lvx %2,%1,%0" :: "b"(vp), "r"( 0), "n"(LO_VEC)); 320 __asm("lvx %2,%1,%0" :: "b"(vp), "r"(16), "n"(HI_VEC)); 321 322 /* 323 * Restore old MSR (AltiVec OFF). 324 */ 325 __asm volatile("sync; mtmsr %0; isync" :: "r"(omsr)); 326 } 327