1/* $NetBSD: trap_subr.S,v 1.13 2020/07/06 10:16:12 rin Exp $ */ 2/*- 3 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects 8 * Agency and which was developed by Matt Thomas of 3am Software Foundry. 9 * 10 * This material is based upon work supported by the Defense Advanced Research 11 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under 12 * Contract No. N66001-09-C-2073. 13 * Approved for Public Release, Distribution Unlimited 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 26 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34 * POSSIBILITY OF SUCH DAMAGE. 35 */ 36 37RCSID("$NetBSD: trap_subr.S,v 1.13 2020/07/06 10:16:12 rin Exp $") 38 39#ifdef _KERNEL_OPT 40#include "opt_altivec.h" 41#include "opt_ddb.h" 42#include "opt_mpc85xx.h" 43#include "opt_multiprocessor.h" 44#endif 45 46 .globl _C_LABEL(sctrapexit), _C_LABEL(trapexit), _C_LABEL(intrcall) 47 48 /* 49 * We have a problem with critical (MSR[CE] or machine check (MSR[ME]) 50 * or debug (MSR[DE]) interrupts/exception in that they could happen 51 * inbetween the mtsprg1 %r2 and mfsprg1 %r2. If that happens, %r2 52 * will be lost. Even if we moved to a different sprg, subsequent 53 * expceptions would use SPRG1 and its value would be lost. The only 54 * way to be safe for CE/ME/DE faults to save and restore SPRG1. 55 * 56 * Since CE/ME/DE faults may happen anytime, we need r1 to always 57 * contain a valid kernel stack pointer. Therefore we use r2 as 58 * our temporary register. 59 * 60 * To prevent %r2 being overwritten, each "level" (normal, critical, 61 * mchk) uses a unique sprg to save %r2 (sprg1, sprg4, sprg5). 62 * 63 * Since we can't control how many nested exceptions we might get, 64 * we don't use a dedicated save area. Instead we have a upwards 65 * growing "stack" of them; the pointer to which is kept in sprg3. 66 * 67 * To allocate from the stack, one fetches sprg3, adds the amount 68 * needed, saves sprg3, and then refers to the save using a 69 * displacement of -amount. 70 */ 71#define FRAME_EXC_PROLOGUE(start, sprg, srr) \ 72 mt##sprg %r2; /* save r2 */ \ 73 mfsprg3 %r2; /* get save_area pointer */ \ 74 addi %r2,%r2,4*(32-start); \ 75 /* allocate save area */ \ 76 mtsprg3 %r2; /* save updated pointer */ \ 77 stmw %r##start,-4*(32-start)(%r2); \ 78 /* free r24-r31 for use */ \ 79 mf##sprg %r26; /* get saved r2 */ \ 80 mfcr %r27; /* get Condition Register */ \ 81 mfxer %r28; /* get XER */ \ 82 mfspr %r30, SPR_##srr##0; /* get SRR0 */ \ 83 mfspr %r31, SPR_##srr##1 /* get SRR1 */ 84 85#define PROLOGUE_GET_DEAR mfspr %r24, SPR_DEAR 86#define PROLOGUE_GET_ESR mfspr %r25, SPR_ESR 87#define PROLOGUE_GET_SRRS mfsrr0 %r24; \ 88 mfsrr1 %r25 89#define PROLOGUE_GET_SPRG1 mfsprg1 %r29 90#define PROLOGUE_GET_DBSR mfspr %r25, SPR_DBSR 91#define SAVE_ESR stw %r25, FRAME_ESR(%r1) 92#define SAVE_DEAR stw %r24, FRAME_DEAR(%r1) 93#define SAVE_DEAR_ESR SAVE_ESR; SAVE_DEAR 94#define SAVE_SRRS SAVE_DEAR_ESR 95#define SAVE_SPRG1 stw %r29, FRAME_SPRG1(%r1) 96#define SAVE_DBSR stw %r25, FRAME_DBSR(%r1) 97#define SAVE_NOTHING /* nothing */ 98#define RESTORE_SPRG1(r) lwz r, FRAME_SPRG1(%r1); \ 99 mtsprg1 r 100#define RESTORE_SRR0(r) lwz r, FRAME_DEAR(%r1); \ 101 mtsrr0 r 102#define RESTORE_SRR1(r) lwz r, FRAME_ESR(%r1); \ 103 mtsrr1 r 104 105#define FRAME_PROLOGUE \ 106 FRAME_EXC_PROLOGUE(26, sprg1, SRR) 107 108#define FRAME_PROLOGUE_DEAR_ESR \ 109 FRAME_EXC_PROLOGUE(24, sprg1, SRR); \ 110 PROLOGUE_GET_ESR; \ 111 PROLOGUE_GET_DEAR 112 113#define FRAME_PROLOGUE_ESR \ 114 FRAME_EXC_PROLOGUE(25, sprg1, SRR); \ 115 PROLOGUE_GET_ESR 116 117#define FRAME_TLBPROLOGUE \ 118 FRAME_EXC_PROLOGUE(20, sprg1, SRR); \ 119 PROLOGUE_GET_ESR; \ 120 PROLOGUE_GET_DEAR 121 122#define FRAME_INTR_PROLOGUE \ 123 FRAME_EXC_PROLOGUE(26, sprg1, SRR) 124 125/* 126 * These need to save SRR0/SRR1 as well their SRR0/SRR1 in case normal 127 * exceptions happened during their execution. 128 */ 129#define FRAME_CRIT_PROLOGUE \ 130 FRAME_EXC_PROLOGUE(24, sprg4, CSRR); \ 131 PROLOGUE_GET_SPRG1; \ 132 PROLOGUE_GET_SRRS 133 134#define FRAME_MCHK_PROLOGUE \ 135 FRAME_EXC_PROLOGUE(24, sprg5, MCSRR); \ 136 PROLOGUE_GET_SPRG1; \ 137 PROLOGUE_GET_SRRS 138 139#define FRAME_DEBUG_PROLOGUE \ 140 FRAME_EXC_PROLOGUE(24, sprg4, CSRR); \ 141 PROLOGUE_GET_SPRG1; \ 142 PROLOGUE_GET_SRRS 143 144/* 145 * DDB expects to fetch the LR from the previous frame. But it also 146 * expects to be pointing at the instruction after the branch link. Since 147 * we didn't branch, we need to advance it by to fake out DDB. But there's 148 * problem. If the routine is in either its first or last two instructions 149 * (before or after its adjusted its stack pointer), we could possibly 150 * overwrite stored return address. So that stored return address needs to 151 * saved and restored. 152 */ 153#if defined(DDB) 154#define FRAME_SAVE_SRR0_FOR_DDB \ 155 lwz %r29, FRAMELEN+CFRAME_LR(%r1); /* fetch old return address */\ 156 stw %r29, FRAME_CFRAME_LR(%r1); /* save it */ \ 157 addi %r30, %r30, 4; /* point to s the next insn */ \ 158 stw %r30, FRAMELEN+CFRAME_LR(%r1) /* appease ddb stacktrace */ 159#define FRAME_RESTORE_RETURN_ADDRESS \ 160 lwz %r3, FRAME_CFRAME_LR(%r1); /* fetch old return address */ \ 161 stw %r3, FRAMELEN+CFRAME_LR(%r1) /* restore it */ 162#else 163#define FRAME_SAVE_SRR0_FOR_DDB 164#define FRAME_RESTORE_RETURN_ADDRESS 165#endif 166 167#ifdef PPC_HAVE_SPE 168#define FRAME_SAVE_SPEFSCR \ 169 mfspefscr %r0; /* get spefscr */ \ 170 stw %r0, FRAME_SPEFSCR(%r1) /* save into trapframe */ 171#define FRAME_RESTORE_SPEFSCR \ 172 lwz %r0, FRAME_SPEFSCR(%r1); /* fetch from trapframe */ \ 173 mtspefscr %r0 /* save spefscr */ 174#else 175#define FRAME_SAVE_SPEFSCR 176#define FRAME_RESTORE_SPEFSCR 177#endif 178/* 179 * Before the first memory refernence, we must have our state inside registers 180 * since the first memory access might cause an exception which would cause 181 * SRR0/SRR1 and DEAR/ESR to become unrecoverable. CR and XER also need to be 182 * saved early since they will modified by instrction flow. The saved stack 183 * pointer is also critical but LR and CTR can be deferred being saved until 184 * we are actually filling a trapframe. 185 */ 186#define FRAME_EXC_ENTER(exc, tf, start, save_prologue) \ 187 mtcr %r31; /* user mode exception? */ \ 188 mr %r31, %r1; /* save SP (SRR1 is safe in CR) */ \ 189 bf MSR_PR, 1f; /* nope, sp is good */ \ 190 mfsprg2 %r2; /* get curlwp */ \ 191 lwz %r2, L_PCB(%r2); /* get uarea of curlwp */ \ 192 addi %r1, %r2, USPACE-CALLFRAMELEN; \ 193 /* start stack at top of it */ \ 1941: \ 195 stwu %r31, -FRAMELEN(%r1); /* get space for trapframe */ \ 196 stw %r0, FRAME_R0(%r1); /* save r0 */ \ 197 stw %r31, FRAME_R1(%r1); /* save (saved) r1 */ \ 198 stw %r26, FRAME_R2(%r1); /* save (saved) r2 */ \ 199 save_prologue; /* save SPRG1/ESR/DEAR */ \ 200 /* At this point, r26, r29, and r31 have been saved so we */ \ 201 /* can use them for LR, CTR, and SRR1. */ \ 202 mflr %r26; /* get Link Register */ \ 203 mfctr %r29; /* get CTR */ \ 204 mfcr %r31; /* get SRR1 */ \ 205 stmw %r26, FRAME_LR(%r1); /* save LR CR XER CTR SRR0/1 */ \ 206 FRAME_SAVE_SRR0_FOR_DDB; \ 207 mr %r0, %r31; /* save SRR1 for a bit */ \ 208 mfsprg3 %r2; /* get save_area pointer */ \ 209 addi %r2,%r2,-4*(32-start); /* find our save area */ \ 210 lmw %r##start,0(%r2); /* get start-r31 */ \ 211 mtsprg3 %r2; /* save updated pointer */ \ 212 stmw %r3, FRAME_R3(%r1); /* save r2-r31 */ \ 213 /* Now everything has been saved */ \ 214 mr %r31, %r0; /* move SRR1 back to r31 */ \ 215 mfsprg2 %r13; /* put curlwp in r13 */ \ 216 FRAME_SAVE_SPEFSCR; \ 217 li %r7, exc; /* load EXC_* */ \ 218 stw %r7, FRAME_EXC(%r1); /* save into trapframe */ \ 219 addi tf, %r1, FRAME_TF /* get address of trap frame */ 220 221#define FRAME_EXC_EXIT(rfi, srr) \ 222 FRAME_RESTORE_RETURN_ADDRESS; /* restore return address */ \ 223 lmw %r26, FRAME_LR(%r1); /* get LR CR XER CTR SRR0/1 */ \ 224 oris %r31,%r31,PSL_CE@h; \ 225 mtspr SPR_##srr##1, %r31; /* restore SRR1 */ \ 226 mtspr SPR_##srr##0, %r30; /* restore SRR0 */ \ 227 FRAME_RESTORE_SPEFSCR; \ 228 mtctr %r29; /* restore CTR */ \ 229 mtxer %r28; /* restore XER */ \ 230 mtcr %r27; /* restore CR */ \ 231 mtlr %r26; /* restore LR */ \ 232 lmw %r2, FRAME_R2(%r1); /* restore r2-r31 */ \ 233 lwz %r0, FRAME_R0(%r1); /* restore r0 */ \ 234 lwz %r1, FRAME_R1(%r1); /* restore r1 */ \ 235 rfi /* return from interrupt */ 236 237 238#define FRAME_ENTER(exc, tf) \ 239 FRAME_EXC_ENTER(exc, tf, 26, SAVE_NOTHING) 240 241#define FRAME_ENTER_ESR(exc, tf) \ 242 FRAME_EXC_ENTER(exc, tf, 25, SAVE_ESR) 243 244#define FRAME_ENTER_DEAR_ESR(exc, tf) \ 245 FRAME_EXC_ENTER(exc, tf, 24, SAVE_DEAR_ESR) 246 247#define FRAME_EXIT FRAME_EXC_EXIT(rfi, SRR) 248 249#define FRAME_TLBENTER(exc) \ 250 FRAME_EXC_ENTER(exc, %r4, 20, SAVE_DEAR_ESR) 251#define FRAME_TLBEXIT FRAME_EXC_EXIT(rfi, SRR) 252 253#define FRAME_MCHK_ENTER(exc) \ 254 FRAME_EXC_ENTER(exc, %r3, 26, SAVE_SPRG1; SAVE_SRRS) 255#define FRAME_MCHK_EXIT \ 256 RESTORE_SRR0(%r28); \ 257 RESTORE_SRR1(%r27); \ 258 RESTORE_SPRG1(%r26); \ 259 FRAME_EXC_EXIT(rfmci, MCSRR) 260 261#define FRAME_DEBUG_ENTER(exc) \ 262 FRAME_EXC_ENTER(exc, %r4, 26, SAVE_SPRG1; SAVE_SRRS) 263#define FRAME_DEBUG_EXIT \ 264 RESTORE_SPRG1(%r26); FRAME_EXC_EXIT(rfci, CSRR) 265 266#define FRAME_INTR_SP \ 267 bf MSR_PR, 1f; /* nope, sp is good */ \ 268 mfsprg2 %r2; /* get curlwp */ \ 269 lwz %r2, L_PCB(%r2); /* get uarea of curlwp */ \ 270 addi %r1, %r2, USPACE-CALLFRAMELEN; \ 271 /* start stack at top of it */ \ 2721: 273 274#define FRAME_INTR_SP_NEW(sym) \ 275 lis %r2,(sym)@ha; \ 276 addi %r1,%r2,(sym)@l 277 278#define FRAME_INTR_XENTER(exc, start, get_intr_sp, save_prologue) \ 279 mtcr %r31; /* user mode exception? */ \ 280 mr %r31, %r1; /* save SP (SRR1 is safe in CR) */ \ 281 get_intr_sp; /* get kernel stack pointer */ \ 282 stwu %r31, -FRAMELEN(%r1); /* get space for trapframe */ \ 283 stw %r0, FRAME_R0(%r1); /* save r0 */ \ 284 stw %r31, FRAME_R1(%r1); /* save (saved) r1 */ \ 285 stw %r26, FRAME_R2(%r1); /* save (saved) r2 */ \ 286 save_prologue; /* save SPRG1 (maybe) */ \ 287 mflr %r26; /* get LR */ \ 288 mfctr %r29; /* get CTR */ \ 289 mfcr %r31; /* get SRR1 */ \ 290 stmw %r26, FRAME_LR(%r1); /* save LR CR XER CTR SRR0/1 */ \ 291 FRAME_SAVE_SRR0_FOR_DDB; \ 292 stw %r3, FRAME_R3(%r1); /* save r3 */ \ 293 stw %r4, FRAME_R4(%r1); /* save r4 */ \ 294 stw %r5, FRAME_R5(%r1); /* save r5 */ \ 295 stw %r6, FRAME_R6(%r1); /* save r6 */ \ 296 stw %r7, FRAME_R7(%r1); /* save r7 */ \ 297 stw %r8, FRAME_R8(%r1); /* save r8 */ \ 298 stw %r9, FRAME_R9(%r1); /* save r9 */ \ 299 stw %r10, FRAME_R10(%r1); /* save r10 */ \ 300 stw %r11, FRAME_R11(%r1); /* save r11 */ \ 301 stw %r12, FRAME_R12(%r1); /* save r12 */ \ 302 stw %r13, FRAME_R13(%r1); /* save r13 */ \ 303 mfsprg3 %r2; /* get save_area pointer */ \ 304 addi %r2,%r2,-4*(32-start); /* find our save area */ \ 305 lmw %r##start,0(%r2); /* get start-r31 */ \ 306 mtsprg3 %r2; /* save updated pointer */ \ 307 mfsprg2 %r13; /* put curlwp into r13 */ \ 308 li %r7, exc; /* load EXC_* */ \ 309 stw %r7, FRAME_EXC(%r1); /* save into trapframe */ \ 310 addi %r3, %r1, FRAME_TF /* only argument is trapframe */ 311 312#define FRAME_INTR_XEXIT(rfi, srr) \ 313 FRAME_RESTORE_RETURN_ADDRESS; /* restore return address */ \ 314 lwz %r8, FRAME_LR(%r1); /* get LR */ \ 315 lwz %r9, FRAME_CR(%r1); /* get CR */ \ 316 lwz %r10, FRAME_XER(%r1); /* get XER */ \ 317 lwz %r11, FRAME_CTR(%r1); /* get CTR */ \ 318 lwz %r12, FRAME_SRR0(%r1); /* get SRR0 */ \ 319 lwz %r13, FRAME_SRR1(%r1); /* get SRR1 */ \ 320 mtspr SPR_##srr##1, %r13; /* restore SRR1 */ \ 321 mtspr SPR_##srr##0, %r12; /* restore SRR0 */ \ 322 mtctr %r11; /* restore CTR */ \ 323 mtxer %r10; /* restore XER */ \ 324 mtcr %r9; /* restore CR */ \ 325 mtlr %r8; /* restore LR */ \ 326 lwz %r13, FRAME_R13(%r1); /* restore r13 */ \ 327 lwz %r12, FRAME_R12(%r1); /* restore r12 */ \ 328 lwz %r11, FRAME_R11(%r1); /* restore r11 */ \ 329 lwz %r10, FRAME_R10(%r1); /* restore r10 */ \ 330 lwz %r9, FRAME_R9(%r1); /* restore r9 */ \ 331 lwz %r8, FRAME_R8(%r1); /* restore r8 */ \ 332 lwz %r7, FRAME_R7(%r1); /* restore r7 */ \ 333 lwz %r6, FRAME_R6(%r1); /* restore r6 */ \ 334 lwz %r5, FRAME_R5(%r1); /* restore r5 */ \ 335 lwz %r4, FRAME_R4(%r1); /* restore r4 */ \ 336 lwz %r3, FRAME_R3(%r1); /* restore r3 */ \ 337 lwz %r2, FRAME_R2(%r1); /* restore r2 */ \ 338 lwz %r0, FRAME_R0(%r1); /* restore r0 */ \ 339 lwz %r1, FRAME_R1(%r1); /* restore r1 */ \ 340 rfi /* return from interrupt */ 341 342#define FRAME_INTR_ENTER(exc) \ 343 FRAME_INTR_XENTER(exc, 26, FRAME_INTR_SP, SAVE_NOTHING) 344#define FRAME_INTR_EXIT \ 345 FRAME_INTR_XEXIT(rfi, SRR) 346#define FRAME_CRIT_ENTER(exc) \ 347 FRAME_INTR_XENTER(exc, 24, FRAME_INTR_SP, SAVE_SPRG1) 348#define FRAME_WDOG_ENTER(exc, sym) \ 349 FRAME_INTR_XENTER(exc, 24, FRAME_INTR_SP_NEW(sym), SAVE_SPRG1) 350#define FRAME_CRIT_EXIT \ 351 RESTORE_SRR0(%r4); \ 352 RESTORE_SRR1(%r5); \ 353 RESTORE_SPRG1(%r6); \ 354 FRAME_INTR_XEXIT(rfci, CSRR) 355 356#if defined(MULTIPROCESSOR) 357#define FRAME_TLBMISSLOCK \ 358 GET_CPUINFO(%r23); \ 359 ldint %r22, CI_MTX_COUNT(%r23); \ 360 subi %r22, %r22, 1; \ 361 stint %r22, CI_MTX_COUNT(%r23); \ 362 isync; \ 363 cmpwi %r22, 0; \ 364 bne 1f; \ 365 ldint %r22, CI_CPL(%r23); \ 366 stint %r22, CI_MTX_OLDSPL(%r23); \ 3671: lis %r23, _C_LABEL(pmap_tlb_miss_lock)@h; \ 368 ori %r23, %r23, _C_LABEL(pmap_tlb_miss_lock)@l; \ 369 li %r20, MTX_LOCK; \ 3702: lwarx %r22, %r20, %r23; \ 371 cmpwi %r22, __SIMPLELOCK_UNLOCKED; \ 372 beq+ 4f; \ 3733: lwzx %r22, %r20, %r23; \ 374 cmpwi %r22, __SIMPLELOCK_UNLOCKED; \ 375 beq+ 2b; \ 376 b 3b; \ 3774: li %r21, __SIMPLELOCK_LOCKED; \ 378 stwcx. %r21, %r20, %r23; \ 379 bne- 2b; \ 380 isync; \ 381 msync; 382#define FRAME_TLBMISSUNLOCK \ 383 sync; \ 384 lis %r23, _C_LABEL(pmap_tlb_miss_lock)@h; \ 385 ori %r23, %r23, _C_LABEL(pmap_tlb_miss_lock)@l; \ 386 li %r22, __SIMPLELOCK_UNLOCKED; \ 387 stw %r22, MTX_LOCK(%r23); \ 388 isync; \ 389 msync; \ 390 GET_CPUINFO(%r23); \ 391 ldint %r22, CI_MTX_COUNT(%r23); \ 392 addi %r22, %r22, 1; \ 393 stint %r22, CI_MTX_COUNT(%r23); \ 394 isync; 395#else /* !MULTIPROCESSOR */ 396#define FRAME_TLBMISSLOCK 397#define FRAME_TLBMISSUNLOCK 398#endif /* MULTIPROCESSOR */ 399 400 .text 401 .p2align 4 402_C_LABEL(critical_input_vector): 403 /* MSR[ME] is unchanged, all others cleared */ 404 FRAME_CRIT_PROLOGUE /* save SP r26-31 CR LR XER */ 405 FRAME_CRIT_ENTER(EXC_CII) 406 bl _C_LABEL(intr_critintr) /* critintr(tf) */ 407 FRAME_CRIT_EXIT 408 409 .p2align 4 410_C_LABEL(machine_check_vector): 411 /* all MSR bits are cleared */ 412 FRAME_MCHK_PROLOGUE /* save SP r25-31 CR LR XER */ 413 FRAME_MCHK_ENTER(EXC_MCHK) 414 /* 415 * MCAR/MCSR don't need to be saved early since MSR[ME] is cleared 416 * on entry. 417 */ 418 mfspr %r7, SPR_MCAR 419 mfspr %r6, SPR_MCSR 420 stw %r6, FRAME_MCSR(%r1) 421 stw %r7, FRAME_MCAR(%r1) 422 li %r3, T_MACHINE_CHECK 423 bl _C_LABEL(trap) /* trap(T_MACHINE_CHECK, tf) */ 424 FRAME_MCHK_EXIT 425 426 .p2align 4 427_C_LABEL(data_storage_vector): 428 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 429 FRAME_PROLOGUE_DEAR_ESR /* save r2 DEAR ESR r24-31 CR XER SRR */ 430 FRAME_ENTER_DEAR_ESR(EXC_DSI, %r4) 431 li %r3, T_DSI 432 /* FRAME_ENTER leaves SRR1 in %r31 */ 433trapenter: 434trapagain: 435 wrtee %r31 /* restore MSR[EE] */ 436 437 bl _C_LABEL(trap) /* trap(trapcode, tf) */ 438_C_LABEL(trapexit): 439 wrteei 0 /* disable interrupts */ 440# andis. %r0, %r31, PSL_CE@h 441# tweqi %r0, 0 442 andi. %r4, %r31, PSL_PR /* lets look at PSL_PR */ 443 beq trapdone /* if clear, skip to exit */ 444 lwz %r4, L_MD_ASTPENDING(%r13) /* get ast pending */ 445 cmplwi %r4, 0 /* is there an ast pending */ 446 beq+ trapdone /* nope, proceed to exit */ 447 li %r6, EXC_AST /* yes. */ 448 stw %r6, FRAME_EXC(%r1) /* pretend this is an AST */ 449 addi %r4, %r1, FRAME_TF /* get address of trap frame */ 450 li %r3, T_AST 451 b trapagain /* and deal with it */ 452trapdone: 453 FRAME_EXIT 454 455 .p2align 4 456_C_LABEL(instruction_storage_vector): 457 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 458 FRAME_PROLOGUE_ESR /* save ESR r2 r25-31 CR XER SRR0/1 */ 459 FRAME_ENTER_ESR(EXC_ISI, %r4) 460 li %r3, T_ISI 461 b trapenter 462 463 .p2align 4 464_ENTRY(external_input_vector) 465 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 466 FRAME_INTR_PROLOGUE /* save SP r25-31 CR LR XER */ 467 FRAME_INTR_ENTER(EXC_EXI) 468 469 bl _C_LABEL(intr_extintr) 470_C_LABEL(intrcall): 471 GET_CPUINFO(%r6) /* get curcpu() */ 472 lwz %r5, FRAME_SRR1(%r1) /* get saved SRR1 */ 473# andis. %r0, %r5, PSL_CE@h 474# tweqi %r0, 0 475 andi. %r4, %r5, PSL_PR /* lets look at PSL_PR */ 476 beq intrexit /* if clear, skip to exit */ 477 lwz %r4, L_MD_ASTPENDING(%r13) /* get ast pending */ 478 cmplwi %r4, 0 /* is there an ast pending */ 479 beq+ intrexit /* nope, proceed to exit */ 480 stmw %r14, FRAME_R14(%r1) /* save rest of registers */ 481 FRAME_SAVE_SPEFSCR 482 mr %r31, %r5 /* needed for trapagain */ 483 li %r4, EXC_AST /* */ 484 stw %r4, FRAME_EXC(%r1) /* pretend this is an AST */ 485 addi %r4, %r1, FRAME_TF /* get address of trap frame */ 486 li %r3, T_AST 487 b trapagain /* and deal with it */ 488intrexit: 489 FRAME_INTR_EXIT 490 491 .p2align 4 492_C_LABEL(alignment_vector): 493 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 494 FRAME_PROLOGUE_DEAR_ESR /* save SP r25-31 CR LR XER */ 495 FRAME_ENTER_DEAR_ESR(EXC_ALI, %r4) 496 li %r3, T_ALIGNMENT 497 b trapenter 498 499 .p2align 4 500_C_LABEL(program_vector): 501 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 502 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 503 FRAME_ENTER_ESR(EXC_PGM, %r4) 504 li %r3, T_PROGRAM 505 b trapenter 506 507#ifdef SPR_IVOR7 508 .p2align 4 509_C_LABEL(fp_unavailable_vector): 510 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 511 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 512 FRAME_ENTER_ESR(EXC_FPU, %r4) 513 li %r3, T_FP_UNAVAILABLE 514 b trapenter 515#endif 516 517 .p2align 4 518_C_LABEL(system_call_vector): 519 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 520 FRAME_PROLOGUE /* save SP r26-31 CR LR XER */ 521 FRAME_ENTER(EXC_SC, %r3) 522 523 wrteei 1 /* enable interrupts */ 524 lwz %r7, L_PROC(%r13) /* get proc for lwp */ 525 lwz %r8, P_MD_SYSCALL(%r7) /* get syscall */ 526 mtlr %r8 /* need to call indirect */ 527 blrl /* syscall(tf) */ 528_C_LABEL(sctrapexit): 529 wrteei 0 /* disable interrupts */ 530 lwz %r4, L_MD_ASTPENDING(%r13) /* get ast pending */ 531 cmplwi %r4, 0 /* is there an ast pending */ 532 beq+ trapdone /* nope, proceed to exit */ 533 li %r0, EXC_AST /* yes. */ 534 stw %r0, FRAME_EXC(%r1) /* pretend this is an AST */ 535 addi %r4, %r1, FRAME_TF /* get address of trap frame */ 536 li %r3, T_AST 537 b trapenter /* and deal with it */ 538 539#ifdef SPR_IVOR9 540 .p2align 4 541_C_LABEL(ap_unavailable_vector): 542 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 543 FRAME_PROLOGUE /* save SP r25-31 CR LR XER */ 544 FRAME_ENTER(EXC_PGM, %r4) 545 li %r3, T_AP_UNAVAILABLE 546 b trapenter 547#endif 548 549 .p2align 4 550_C_LABEL(decrementer_vector): 551 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 552 FRAME_INTR_PROLOGUE /* save SP r25-31 CR LR XER */ 553 FRAME_INTR_ENTER(EXC_DECR) 554 555 bl _C_LABEL(intr_decrintr) 556 b intrexit 557 558 .p2align 4 559_C_LABEL(fixed_interval_timer_vector): 560 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 561 FRAME_PROLOGUE /* save SP r25-31 CR LR XER */ 562 FRAME_INTR_ENTER(EXC_FIT) 563 564 bl _C_LABEL(intr_fitintr) 565 b intrexit 566 567#ifdef E500_WDOG_STACK 568 .data 569 .lcomm wdogstk,4096 570#endif 571 .text 572 .p2align 4 573_C_LABEL(watchdog_timer_vector): 574 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 575 FRAME_CRIT_PROLOGUE /* save SP r25-31 CR LR XER */ 576#ifdef E500_WDOG_STACK 577 FRAME_WDOG_ENTER(EXC_WDOG, wdogstk+4096-CALLFRAMELEN) 578#else 579 FRAME_CRIT_ENTER(EXC_WDOG); 580#endif 581 582 bl _C_LABEL(intr_wdogintr) 583 FRAME_CRIT_EXIT 584 585 .p2align 4 586_C_LABEL(data_tlb_error_vector): 587 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 588 FRAME_TLBPROLOGUE 589 FRAME_TLBMISSLOCK 590 /* 591 * Registers as this point: 592 * 593 * r2 = cpu_info 594 * r20 = scratch 595 * r21 = scratch 596 * r22 = scratch 597 * r23 = scratch 598 * r24 = DEAR 599 * r25 = ESR 600 * r26 = saved r2 601 * r27 = CR 602 * r28 = XER 603 * r29 = scratch 604 * r30 = SRR0 605 * r31 = SRR1 606 * 607 * Except for r29, these values must be retained. However we must 608 * be cognizant of nesting. There are two cases here, both related. 609 * 610 * We get a critical input or machine check exception and the kernel 611 * stack doesn't have a TLB entry so we take an exception. The other 612 * nesting path is some page used by the exception handler will cause 613 * a TLB data error. 614 * 615 * The second case (more probable) is that the PTE loading will fail 616 * so we will have to do a hard trap to resolve it. But in doing so 617 * we need to save a trapframe which could result in another DTLB 618 * fault. 619 * 620 * In all cases, the save area stack shall protect us. 621 */ 622 /* 623 * Attempt to update the TLB from the page table. 624 */ 625 mflr %r29 /* save LR */ 626 mr %r23, %r24 /* address of exception */ 627 rlwinm %r22, %r31, /* index into ci_pmap_segtab */\ 628 MSR_DS+PTR_SCALESHIFT+1, \ 629 31-PTR_SCALESHIFT, \ 630 31-PTR_SCALESHIFT /* move PSL_DS[27] to bit 29 */ 631 bl pte_load 632 FRAME_TLBMISSUNLOCK 633 mtlr %r29 /* restore LR */ 634 /* 635 * If we returned, pte load failed so let trap deal with it but 636 * has kept the contents of r24-r31 (expect r29) intact. 637 */ 638 FRAME_TLBENTER(EXC_DSI) 639 li %r3, T_DATA_TLB_ERROR 640 b trapenter 641 642 .p2align 4 643_C_LABEL(instruction_tlb_error_vector): 644 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 645 FRAME_TLBPROLOGUE 646 FRAME_TLBMISSLOCK 647 /* 648 * Attempt to update the TLB from the page table. 649 */ 650 mflr %r29 /* save LR */ 651 mr %r23, %r30 /* PC of exception */ 652 rlwinm %r22, %r31, /* index into ci_pmap_segtab */\ 653 MSR_IS+PTR_SCALESHIFT+1, \ 654 31-PTR_SCALESHIFT, \ 655 31-PTR_SCALESHIFT /* move PSL_IS[26] to bit 29 */ 656 bl pte_load 657 FRAME_TLBMISSUNLOCK 658 mtlr %r29 /* restore LR */ 659 /* 660 * If we returned, pte load failed so let trap deal with it but 661 * has kept the contents of r24-r31 (expect r29) intact. 662 */ 663 FRAME_TLBENTER(EXC_ISI) 664 li %r3, T_INSTRUCTION_TLB_ERROR 665 b trapenter 666 667 .p2align 4 668_C_LABEL(debug_vector): 669 FRAME_CRIT_PROLOGUE /* save SP r25-31 CR LR XER */ 670 FRAME_CRIT_ENTER(EXC_DEBUG) 671 mfspr %r6, SPR_DBSR 672 stw %r6, FRAME_ESR(%r1) 673 li %r3, T_DEBUG 674 bl _C_LABEL(trap) 675 FRAME_CRIT_EXIT 676 677 .p2align 4 678_C_LABEL(spv_unavailable_vector): 679 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 680 FRAME_ENTER_ESR(EXC_VEC, %r4) 681 li %r3, T_SPE_UNAVAILABLE 682 b trapenter 683 684 .p2align 4 685_C_LABEL(fpdata_vector): 686 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 687 FRAME_ENTER_ESR(EXC_FPA, %r4) 688 li %r3, T_EMBEDDED_FP_DATA 689 b trapenter 690 691 .p2align 4 692_C_LABEL(fpround_vector): 693 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 694 FRAME_ENTER_ESR(EXC_FPA, %r4) 695 li %r3, T_EMBEDDED_FP_ROUND 696 b trapenter 697 698 .p2align 4 699_C_LABEL(perfmon_vector): 700 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 701 FRAME_ENTER_ESR(EXC_PERF, %r4) 702 li %r3, T_EMBEDDED_PERF_MONITOR 703 b trapenter 704 705 .p2align 4 706pte_load: 707 /* 708 * r2 = scratch 709 * r20 = scratch 710 * r21 = scratch 711 * r22 = index into ci_pmap_{kern,user}_segtab 712 * r23 = faulting address 713 * The rest are for reference and aren't modifiable. If the load 714 * fails, they will be used by FRAME_TLBENTER to create the trapframe. 715 * r24 = DEAR 716 * r25 = ESR 717 * r26 = saved r2 718 * r27 = CR 719 * r28 = XER 720 * r29 = LR 721 * r30 = SRR0 722 * r31 = SRR1 723 */ 724 cmplwi %cr2, %r22, 0 /* remember address space */ 725 GET_CPUINFO(%r2) 726 addi %r22, %r22, CI_PMAP_SEGTAB /* index into segtab(s) */ 727 lwzx %r20, %r22, %r2 /* load kern/user L1 PT addr */ 728 cmplwi %r20, 0 /* is segtab null? */ 729 beqlr %cr0 /* yes, return to fallback to trap */ 730 731 rlwinm %r22, %r23, NSEGPG_SCALESHIFT + PTR_SCALESHIFT, \ 732 31-(NSEGPG_SCALESHIFT + PTR_SCALESHIFT - 1), \ 733 31-PTR_SCALESHIFT /* extract addr bits [0:9] to [20:29] */ 734 lwzx %r20, %r22, %r20 /* load address of page table page */ 735 cmplwi %r20, 0 /* is page null? */ 736 beqlr %cr0 /* yes, return to fallback to trap */ 737 738 rlwinm %r22, %r23, \ 739 NSEGPG_SCALESHIFT + NPTEPG_SCALESHIFT + PTE_SCALESHIFT, \ 740 31-(NPTEPG_SCALESHIFT + PTE_SCALESHIFT - 1), \ 741 31-PTE_SCALESHIFT /* extract addr bits [10:19] to [20:29] */ 742 lwzx %r20, %r22, %r20 /* load PTE from page table page */ 743 cmplwi %r20, 0 /* is there a valid PTE? */ 744 beqlr %cr0 /* no, return to fallback to trap */ 745 746#if (PTE_UNSYNCED << 1) != PTE_xX 747#error PTE_UNSYNCED definition error 748#endif 749#if (PTE_UNMODIFIED << 1) != PTE_xW 750#error PTE_UNMODIFIED definition error 751#endif 752 andi. %r22, %r20, (PTE_UNSYNCED|PTE_UNMODIFIED) 753 /* Does the PTE need to be changed? */ 754 rotlwi %r22, %r22, 1 /* if so, clear the right PTE bits */ 755 andc %r20, %r20, %r22 /* pte &= ~((pte & (PTE_UNSYNCED|PTE_UNMODIFIED)) << 1)*/ 756 757 /* 758 * r24-r32 = (no touch) 759 * r23 = scratch (was fault addr) 760 * r22 = scratch 761 * r21 = scratch 762 * r20 = pte 763 * cr2 = AS 0=eq/!0=ne 764 */ 765 766 /* 767 * This is all E500 specific. We should have a patchable branch 768 * to support other BookE (440) implementations. 769 */ 770e500_pte_load: 771 bne+ %cr2, 1f /* user access? MAS1 is ok. */ 772 mfspr %r22, SPR_MAS1 /* get MAS1 */ 773 lis %r21, MAS1_TID@h /* get TID mask */ 774 andc %r22, %r22, %r21 /* clear TID */ 775 mtspr SPR_MAS1, %r22 /* save MAS1 */ 7761: 777 andi. %r21, %r20, PTE_WIMGE_MASK /* extract WIMGE from PTE */ 778 cmplwi %r21, PTE_M /* if just PTE_M is set, */ 779 beq+ %cr0, 2f /* skip munging mas2 */ 780 mfspr %r22, SPR_MAS2 /* get MAS2 (updated by error) */ 781 clrrwi %r22, %r22, PTE_RWX_SHIFT /* clear WIMGE bits */ 782 or %r22, %r22, %r21 /* combine with MAS2 contents */ 783 mtspr SPR_MAS2, %r22 /* put back into MAS2 */ 7842: 785 /* 786 * r23 = fault addr 787 * r22 = scratch 788 * r21 = scratch 789 * r20 = pte 790 */ 791 792 /* 793 * In MAS3, the protection bits are in the low 6 bits: 794 * UX SX UW SW UR SR 795 * The User bits are 1 bit left of their Supervisor counterparts. 796 * Rotate the PTE protection bits left until they wrap around to become 797 * the least significant bits, where the Supervisor protection bits 798 * are located. Increase the rotate amount by 1 to place them where 799 * the User protection bits are located. We get that 1 by extracting 800 * the MAS1[TS] (set for User access) and moving it to bit 31 (LSB). 801 */ 802 mfspr %r21, SPR_MAS1 /* get MAS1 which has TS bit */ 803 extrwi %r21, %r21, 1, 31-MAS1_TS_SHIFT 804 /* extract MAS1_TS to LSB */ 805 clrrwi %r23, %r20, PAGE_SHIFT /* clear non-RPN bits from PTE */ 806 andi. %r20, %r20, PTE_RWX_MASK /* isolate protection bits */ 807 rotrwi %r20, %r20, PTE_RWX_SHIFT 808 andi. %r22, %r20, (MAS3_SW|MAS3_SR) /* user pages need to be R/W by kernel */ 809 rotlw %r20, %r20, %r21 /* rotate protection to correct loc */ 810 or %r20, %r20, %r22 /* combine system protection bits */ 811 or %r23, %r23, %r20 /* combine RPN and protection bits */ 812 mtspr SPR_MAS3, %r23 /* put into MAS3 */ 813 isync /* because ECORE500RM tells us too */ 814 tlbwe /* write the TLB entry */ 815 /* 816 * Increment a counter to show how many tlb misses we've handled here. 817 */ 818 lmw %r30, CI_EV_TLBMISS_SOFT(%r2) 819 addic %r31, %r31, 1 820 addze %r30, %r30 821 stmw %r30, CI_EV_TLBMISS_SOFT(%r2) 822 823 FRAME_TLBMISSUNLOCK 824 825 /* 826 * Cleanup and leave. We know any higher priority exception will 827 * save and restore SPRG1 and %r2 thereby preserving their values. 828 * 829 * r24 = DEAR (don't care) 830 * r25 = ESR (don't care) 831 * r26 = saved r2 832 * r27 = CR 833 * r28 = XER 834 * r29 = LR 835 * r30 = LSW of counter 836 * r31 = MSW of counter 837 */ 838 mtlr %r29 /* restore Link Register */ 839 mtxer %r28 /* restore XER */ 840 mtcr %r27 /* restore Condition Register */ 841 mtsprg1 %r26 /* save saved r2 across load multiple */ 842 mfsprg3 %r2 /* get end of save area */ 843 addi %r2,%r2,-4*(32-20) /* adjust save area down */ 844 lmw %r20,0(%r2) /* restore r20-r31 */ 845 mtsprg3 %r2 /* save new end of save area */ 846 mfsprg1 %r2 /* restore r2 */ 847 rfi 848 849 .p2align 4 850 .globl _C_LABEL(exception_init) 851_C_LABEL(exception_init): 852 lis %r6,_C_LABEL(critical_input_vector)@h 853 mtspr SPR_IVPR, %r6 854 855 ori %r5,%r6,_C_LABEL(critical_input_vector)@l 856 mtspr SPR_IVOR0, %r5 857 858 ori %r5,%r6,_C_LABEL(machine_check_vector)@l 859 mtspr SPR_IVOR1, %r5 860 861 ori %r5,%r6,_C_LABEL(data_storage_vector)@l 862 mtspr SPR_IVOR2, %r5 863 864 ori %r5,%r6,_C_LABEL(instruction_storage_vector)@l 865 mtspr SPR_IVOR3, %r5 866 867 ori %r5,%r6,_C_LABEL(external_input_vector)@l 868 mtspr SPR_IVOR4, %r5 869 870 ori %r5,%r6,_C_LABEL(alignment_vector)@l 871 mtspr SPR_IVOR5, %r5 872 873 ori %r5,%r6,_C_LABEL(program_vector)@l 874 mtspr SPR_IVOR6, %r5 875 876#ifdef SPR_IVOR7 877 ori %r5,%r6,_C_LABEL(fp_unavailable_vector)@l 878 mtspr SPR_IVOR7, %r5 879#endif 880 881 ori %r5,%r6,_C_LABEL(system_call_vector)@l 882 mtspr SPR_IVOR8, %r5 883 884#ifdef SPR_IVOR9 885 ori %r5,%r6,_C_LABEL(ap_unavailable_vector)@l 886 mtspr SPR_IVOR9, %r5 887#endif 888 889 ori %r5,%r6,_C_LABEL(decrementer_vector)@l 890 mtspr SPR_IVOR10, %r5 891 892 ori %r5,%r6,_C_LABEL(fixed_interval_timer_vector)@l 893 mtspr SPR_IVOR11, %r5 894 895 ori %r5,%r6,_C_LABEL(watchdog_timer_vector)@l 896 mtspr SPR_IVOR12, %r5 897 898 ori %r5,%r6,_C_LABEL(data_tlb_error_vector)@l 899 mtspr SPR_IVOR13, %r5 900 901 ori %r5,%r6,_C_LABEL(instruction_tlb_error_vector)@l 902 mtspr SPR_IVOR14, %r5 903 904 ori %r5,%r6,_C_LABEL(debug_vector)@l 905 mtspr SPR_IVOR15, %r5 906 907 ori %r5,%r6,_C_LABEL(spv_unavailable_vector)@l 908 mtspr SPR_IVOR32, %r5 909 910 ori %r5,%r6,_C_LABEL(fpdata_vector)@l 911 mtspr SPR_IVOR33, %r5 912 913 ori %r5,%r6,_C_LABEL(fpround_vector)@l 914 mtspr SPR_IVOR34, %r5 915 916 ori %r5,%r6,_C_LABEL(perfmon_vector)@l 917 mtspr SPR_IVOR35, %r5 918 919 mfspr %r5, SPR_PIR /* get Processor ID register */ 920 cmplwi %r5,0 921 bnelr /* return if non-0 (non-primary) */ 922 923 lis %r5,_C_LABEL(powerpc_intrsw)@ha 924 stw %r3,_C_LABEL(powerpc_intrsw)@l(%r5) 925 926 blr 927 928#ifdef notyet 929 .data 930 .lcomm ddbstk,4096 931 .text 932 933_ENTRY(cpu_Debugger) 934 mflr %r0 935 stw %r0, CFRAME_LR(%r1) 936 937 mfmsr %r3 938 wrteei 0 939 mr %r4,%r1 940 lis %r10,ddbstk@ha 941 addi %r10,%r10,ddbstk@l 942 sub %r5,%r1,%r10 943 cmplwi %r5,4096 944 blt %cr0, 1f 945 addi %r1,%r10,4096-CALLFRAMELEN 9461: 947 stwu %r4,-FRAMELEN(%r1) 948 stw %r4,FRAME_R1(%r1) 949 stmw %r13,FRAME_R13(%r1) 950 mr %r26,%r0 951 mfcr %r27 952 mfxer %r28 953 mfctr %r29 954 mr %r30,%r0 955 mr %r31,%r3 956 stmw %r26,FRAME_LR(%r1) 957 mr %r31,%r1 958 mr %r1,%r10 959 addi %r4,%r1,FRAME_TF 960 li %r3,EXC_PGM 961 stw %r3,FRAME_EXC(%r1) 962 li %r3,T_PROGRAM 963 bl _C_LABEL(trap) 964 lmw %r26,FRAME_LR(%r1) 965 mtlr %r26 966 mtcr %r27 967 mtxer %r28 968 mtctr %r29 969 mr %r0,%r31 970 lmw %r13,FRAME_R13(%r1) 971 lwz %r1,FRAME_R1(%r1) 972 wrtee %r0 973 blr 974#endif /* notyet */ 975