1/* $NetBSD: trap_subr.S,v 1.6 2011/06/21 05:33:04 matt Exp $ */ 2/*- 3 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects 8 * Agency and which was developed by Matt Thomas of 3am Software Foundry. 9 * 10 * This material is based upon work supported by the Defense Advanced Research 11 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under 12 * Contract No. N66001-09-C-2073. 13 * Approved for Public Release, Distribution Unlimited 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 26 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34 * POSSIBILITY OF SUCH DAMAGE. 35 */ 36 37RCSID("$NetBSD: trap_subr.S,v 1.6 2011/06/21 05:33:04 matt Exp $") 38 39 .globl _C_LABEL(sctrapexit), _C_LABEL(trapexit), _C_LABEL(intrcall) 40 41 /* 42 * We have a problem with critical (MSR[CE] or machine check (MSR[ME]) 43 * or debug (MSR[DE]) interrupts/exception in that they could happen 44 * inbewtween the mtsprg1 %r2 and mfsprg1 %r2. If that happens, %r2 45 * will be lost. Even if we moved to a different sprg, subsequent 46 * expceptions would use SPRG1 and its value would be lost. The only 47 * way to be safe for CE/ME/DE faults to save and restore SPRG1. 48 * 49 * Since CE/ME/DE faults may happen anytime, we need r1 to always 50 * contain a valid kernel stack pointer. Therefore we use r2 as 51 * our temporary register. 52 * 53 * To prevent %r2 being overwritten, each "level" (normal, critical, 54 * mchk) uses a unique sprg to save %r2 (sprg1, sprg4, sprg5). 55 * 56 * Since we can't control how many nested exceptions we might get, 57 * we don't use a dedicated save area. Instead we have a upwards 58 * growing "stack" of them; the pointer to which is kept in sprg3. 59 * 60 * To allocate from the stack, one fetches sprg3, adds the amount 61 * needed, saves sprg3, and then refers to the save using a 62 * displacement of -amount. 63 */ 64#define FRAME_EXC_PROLOGUE(start, sprg, srr) \ 65 mt##sprg %r2; /* save r2 */ \ 66 mfsprg3 %r2; /* get save_area pointer */ \ 67 addi %r2,%r2,4*(32-start); \ 68 /* allocate save area */ \ 69 mtsprg3 %r2; /* save updated pointer */ \ 70 stmw %r##start,-4*(32-start)(%r2); \ 71 /* free r24-r31 for use */ \ 72 mf##sprg %r26; /* get saved r2 */ \ 73 mfcr %r27; /* get Condition Register */ \ 74 mfxer %r28; /* get XER */ \ 75 mfspr %r30, SPR_##srr##0; /* get SRR0 */ \ 76 mfspr %r31, SPR_##srr##1 /* get SRR1 */ 77 78#define PROLOGUE_GET_DEAR mfspr %r24, SPR_DEAR 79#define PROLOGUE_GET_ESR mfspr %r25, SPR_ESR 80#define PROLOGUE_GET_SRRS mfsrr0 %r24; \ 81 mfsrr1 %r25 82#define PROLOGUE_GET_SPRG1 mfsprg1 %r29 83#define PROLOGUE_GET_DBSR mfspr %r25, SPR_DBSR 84#define SAVE_ESR stw %r25, FRAME_ESR(%r1) 85#define SAVE_DEAR stw %r24, FRAME_DEAR(%r1) 86#define SAVE_DEAR_ESR SAVE_ESR; SAVE_DEAR 87#define SAVE_SRRS SAVE_DEAR_ESR 88#define SAVE_SPRG1 stw %r29, FRAME_SPRG1(%r1) 89#define SAVE_DBSR stw %r25, FRAME_DBSR(%r1) 90#define SAVE_NOTHING /* nothing */ 91#define RESTORE_SPRG1(r) lwz r, FRAME_SPRG1(%r1); \ 92 mtsprg1 r 93#define RESTORE_SRR0(r) lwz r, FRAME_DEAR(%r1); \ 94 mtsrr0 r 95#define RESTORE_SRR1(r) lwz r, FRAME_ESR(%r1); \ 96 mtsrr1 r 97 98#define FRAME_PROLOGUE \ 99 FRAME_EXC_PROLOGUE(26, sprg1, SRR) 100 101#define FRAME_PROLOGUE_DEAR_ESR \ 102 FRAME_EXC_PROLOGUE(24, sprg1, SRR); \ 103 PROLOGUE_GET_ESR; \ 104 PROLOGUE_GET_DEAR 105 106#define FRAME_PROLOGUE_ESR \ 107 FRAME_EXC_PROLOGUE(25, sprg1, SRR); \ 108 PROLOGUE_GET_ESR 109 110#define FRAME_TLBPROLOGUE \ 111 FRAME_EXC_PROLOGUE(20, sprg1, SRR); \ 112 PROLOGUE_GET_ESR; \ 113 PROLOGUE_GET_DEAR 114 115#define FRAME_INTR_PROLOGUE \ 116 FRAME_EXC_PROLOGUE(26, sprg1, SRR) 117 118/* 119 * These need to save SRR0/SRR1 as well their SRR0/SRR1 in case normal 120 * exceptions happened during their execution. 121 */ 122#define FRAME_CRIT_PROLOGUE \ 123 FRAME_EXC_PROLOGUE(24, sprg4, CSRR); \ 124 PROLOGUE_GET_SPRG1; \ 125 PROLOGUE_GET_SRRS 126 127#define FRAME_MCHK_PROLOGUE \ 128 FRAME_EXC_PROLOGUE(24, sprg5, MCSRR); \ 129 PROLOGUE_GET_SPRG1; \ 130 PROLOGUE_GET_SRRS 131 132#define FRAME_DEBUG_PROLOGUE \ 133 FRAME_EXC_PROLOGUE(24, sprg4, CSRR); \ 134 PROLOGUE_GET_SPRG1; \ 135 PROLOGUE_GET_SRRS 136 137/* 138 * DDB expects to fetch the LR from the previous frame. But it also 139 * expects to be pointing at the instruction after the branch link. Since 140 * we didn't branch, we need to advance it by to fake out DDB. 141 */ 142#ifdef DDB 143#define FRAME_SAVE_SRR0_FOR_DDB \ 144 addi %r30, %r30, 4; /* ddb thinks its the next insn */ \ 145 stw %r30, FRAMELEN+SZREG(%r1); /* appease ddb stacktrace */ 146#else 147#define FRAME_SAVE_SRR0_FOR_DDB 148#endif 149 150#ifdef PPC_HAVE_SPE 151#define FRAME_SAVE_SPEFSCR \ 152 mfspefscr %r0; /* get spefscr */ \ 153 stw %r0, FRAME_SPEFSCR(%r1) /* save into trapframe */ 154#define FRAME_RESTORE_SPEFSCR \ 155 lwz %r0, FRAME_SPEFSCR(%r1); /* fetch from trapframe */ \ 156 mtspefscr %r0 /* save spefscr */ 157#else 158#define FRAME_SAVE_SPEFSCR 159#define FRAME_RESTORE_SPEFSCR 160#endif 161/* 162 * Before the first memory refernence, we must have our state inside registers 163 * since the first memory access might cause an exception which would cause 164 * SRR0/SRR1 and DEAR/ESR to become unrecoverable. CR and XER also need to be 165 * saved early since they will modified by instrction flow. The saved stack 166 * pointer is also critical but LR and CTR can be deferred being saved until 167 * we are actually filling a trapframe. 168 */ 169#define FRAME_EXC_ENTER(exc, tf, start, save_prologue) \ 170 mtcr %r31; /* user mode exception? */ \ 171 mr %r31, %r1; /* save SP (SRR1 is safe in CR) */ \ 172 bf MSR_PR, 1f; /* nope, sp is good */ \ 173 mfsprg2 %r2; /* get curlwp */ \ 174 lwz %r2, L_PCB(%r2); /* get uarea of curlwp */ \ 175 addi %r1, %r2, USPACE-CALLFRAMELEN; \ 176 /* start stack at top of it */ \ 1771: \ 178 stw %r30, CFRAME_LR(%r1); /* save in previous callframe */ \ 179 stwu %r31, -FRAMELEN(%r1); /* get space for trapframe */ \ 180 stw %r0, FRAME_R0(%r1); /* save r0 */ \ 181 stw %r31, FRAME_R1(%r1); /* save (saved) r1 */ \ 182 stw %r26, FRAME_R2(%r1); /* save (saved) r2 */ \ 183 save_prologue; /* save SPRG1/ESR/DEAR */ \ 184 /* At this point, r26, r29, and r31 have been saved so we */ \ 185 /* can use them for LR, CTR, and SRR1. */ \ 186 mflr %r26; /* get Link Register */ \ 187 mfctr %r29; /* get CTR */ \ 188 mfcr %r31; /* get SRR1 */ \ 189 stmw %r26, FRAME_LR(%r1); /* save LR CR XER CTR SRR0/1 */ \ 190 FRAME_SAVE_SRR0_FOR_DDB; \ 191 mr %r0, %r31; /* save SRR1 for a bit */ \ 192 mfsprg3 %r2; /* get save_area pointer */ \ 193 addi %r2,%r2,-4*(32-start); /* find our save area */ \ 194 lmw %r##start,0(%r2); /* get start-r31 */ \ 195 mtsprg3 %r2; /* save updated pointer */ \ 196 stmw %r3, FRAME_R3(%r1); /* save r2-r31 */ \ 197 /* Now everything has been saved */ \ 198 mr %r31, %r0; /* move SRR1 back to r31 */ \ 199 mfsprg2 %r13; /* put curlwp in r13 */ \ 200 FRAME_SAVE_SPEFSCR; \ 201 li %r7, exc; /* load EXC_* */ \ 202 stw %r7, FRAME_EXC(%r1); /* save into trapframe */ \ 203 addi tf, %r1, FRAME_TF /* get address of trap frame */ 204 205#define FRAME_EXC_EXIT(rfi, srr) \ 206 lmw %r26, FRAME_LR(%r1); /* get LR CR XER CTR SRR0/1 */ \ 207 oris %r31,%r31,PSL_CE@h; \ 208 mtspr SPR_##srr##1, %r31; /* restore SRR1 */ \ 209 mtspr SPR_##srr##0, %r30; /* restore SRR0 */ \ 210 FRAME_RESTORE_SPEFSCR; \ 211 mtctr %r29; /* restore CTR */ \ 212 mtxer %r28; /* restore XER */ \ 213 mtcr %r27; /* restore CR */ \ 214 mtlr %r26; /* restore LR */ \ 215 lmw %r2, FRAME_R2(%r1); /* restore r2-r31 */ \ 216 lwz %r0, FRAME_R0(%r1); /* restore r0 */ \ 217 lwz %r1, FRAME_R1(%r1); /* restore r1 */ \ 218 rfi /* return from interrupt */ 219 220 221#define FRAME_ENTER(exc, tf) \ 222 FRAME_EXC_ENTER(exc, tf, 26, SAVE_NOTHING) 223 224#define FRAME_ENTER_ESR(exc, tf) \ 225 FRAME_EXC_ENTER(exc, tf, 25, SAVE_ESR) 226 227#define FRAME_ENTER_DEAR_ESR(exc, tf) \ 228 FRAME_EXC_ENTER(exc, tf, 24, SAVE_DEAR_ESR) 229 230#define FRAME_EXIT FRAME_EXC_EXIT(rfi, SRR) 231 232#define FRAME_TLBENTER(exc) \ 233 FRAME_EXC_ENTER(exc, %r4, 20, SAVE_DEAR_ESR) 234#define FRAME_TLBEXIT FRAME_EXC_EXIT(rfi, SRR) 235 236#define FRAME_MCHK_ENTER(exc) \ 237 FRAME_EXC_ENTER(exc, %r3, 26, SAVE_SPRG1; SAVE_SRRS) 238#define FRAME_MCHK_EXIT \ 239 RESTORE_SRR0(%r28); \ 240 RESTORE_SRR1(%r27); \ 241 RESTORE_SPRG1(%r26); \ 242 FRAME_EXC_EXIT(rfmci, MCSRR) 243 244#define FRAME_DEBUG_ENTER(exc) \ 245 FRAME_EXC_ENTER(exc, %r4, 26, SAVE_SPRG1; SAVE_SRRS) 246#define FRAME_DEBUG_EXIT \ 247 RESTORE_SPRG1(%r26); FRAME_EXC_EXIT(rfci, CSRR) 248 249#define FRAME_INTR_SP \ 250 bf MSR_PR, 1f; /* nope, sp is good */ \ 251 mfsprg2 %r2; /* get curlwp */ \ 252 lwz %r2, L_PCB(%r2); /* get uarea of curlwp */ \ 253 addi %r1, %r2, USPACE-CALLFRAMELEN; \ 254 /* start stack at top of it */ \ 2551: 256 257#define FRAME_INTR_SP_NEW(sym) \ 258 lis %r2,(sym)@ha; \ 259 addi %r1,%r2,(sym)@l 260 261#define FRAME_INTR_XENTER(exc, start, get_intr_sp, save_prologue) \ 262 mtcr %r31; /* user mode exception? */ \ 263 mr %r31, %r1; /* save SP (SRR1 is safe in CR) */ \ 264 get_intr_sp; /* get kernel stack pointer */ \ 265 stw %r30, CFRAME_LR(%r1); /* save in .. */ \ 266 stwu %r31, -FRAMELEN(%r1); /* get space for trapframe */ \ 267 stw %r0, FRAME_R0(%r1); /* save r0 */ \ 268 stw %r31, FRAME_R1(%r1); /* save (saved) r1 */ \ 269 stw %r26, FRAME_R2(%r1); /* save (saved) r2 */ \ 270 save_prologue; /* save SPRG1 (maybe) */ \ 271 mflr %r26; /* get LR */ \ 272 mfctr %r29; /* get CTR */ \ 273 mfcr %r31; /* get SRR1 */ \ 274 stmw %r26, FRAME_LR(%r1); /* save LR CR XER CTR SRR0/1 */ \ 275 FRAME_SAVE_SRR0_FOR_DDB; \ 276 stw %r3, FRAME_R3(%r1); /* save r3 */ \ 277 stw %r4, FRAME_R4(%r1); /* save r4 */ \ 278 stw %r5, FRAME_R5(%r1); /* save r5 */ \ 279 stw %r6, FRAME_R6(%r1); /* save r6 */ \ 280 stw %r7, FRAME_R7(%r1); /* save r7 */ \ 281 stw %r8, FRAME_R8(%r1); /* save r8 */ \ 282 stw %r9, FRAME_R9(%r1); /* save r9 */ \ 283 stw %r10, FRAME_R10(%r1); /* save r10 */ \ 284 stw %r11, FRAME_R11(%r1); /* save r11 */ \ 285 stw %r12, FRAME_R12(%r1); /* save r12 */ \ 286 stw %r13, FRAME_R13(%r1); /* save r13 */ \ 287 mfsprg3 %r2; /* get save_area pointer */ \ 288 addi %r2,%r2,-4*(32-start); /* find our save area */ \ 289 lmw %r##start,0(%r2); /* get start-r31 */ \ 290 mtsprg3 %r2; /* save updated pointer */ \ 291 mfsprg2 %r13; /* put curlwp into r13 */ \ 292 li %r7, exc; /* load EXC_* */ \ 293 stw %r7, FRAME_EXC(%r1); /* save into trapframe */ \ 294 addi %r3, %r1, FRAME_TF /* only argument is trapframe */ 295 296#define FRAME_INTR_XEXIT(rfi, srr) \ 297 lwz %r8, FRAME_LR(%r1); /* get LR */ \ 298 lwz %r9, FRAME_CR(%r1); /* get CR */ \ 299 lwz %r10, FRAME_XER(%r1); /* get XER */ \ 300 lwz %r11, FRAME_CTR(%r1); /* get CTR */ \ 301 lwz %r12, FRAME_SRR0(%r1); /* get SRR0 */ \ 302 lwz %r13, FRAME_SRR1(%r1); /* get SRR1 */ \ 303 mtspr SPR_##srr##1, %r13; /* restore SRR1 */ \ 304 mtspr SPR_##srr##0, %r12; /* restore SRR0 */ \ 305 mtctr %r11; /* restore CTR */ \ 306 mtxer %r10; /* restore XER */ \ 307 mtcr %r9; /* restore CR */ \ 308 mtlr %r8; /* restore LR */ \ 309 lwz %r13, FRAME_R13(%r1); /* restore r13 */ \ 310 lwz %r12, FRAME_R12(%r1); /* restore r12 */ \ 311 lwz %r11, FRAME_R11(%r1); /* restore r11 */ \ 312 lwz %r10, FRAME_R10(%r1); /* restore r10 */ \ 313 lwz %r9, FRAME_R9(%r1); /* restore r9 */ \ 314 lwz %r8, FRAME_R8(%r1); /* restore r8 */ \ 315 lwz %r7, FRAME_R7(%r1); /* restore r7 */ \ 316 lwz %r6, FRAME_R6(%r1); /* restore r6 */ \ 317 lwz %r5, FRAME_R5(%r1); /* restore r5 */ \ 318 lwz %r4, FRAME_R4(%r1); /* restore r4 */ \ 319 lwz %r3, FRAME_R3(%r1); /* restore r3 */ \ 320 lwz %r2, FRAME_R2(%r1); /* restore r2 */ \ 321 lwz %r0, FRAME_R0(%r1); /* restore r0 */ \ 322 lwz %r1, FRAME_R1(%r1); /* restore r1 */ \ 323 rfi /* return from interrupt */ 324 325#define FRAME_INTR_ENTER(exc) \ 326 FRAME_INTR_XENTER(exc, 26, FRAME_INTR_SP, SAVE_NOTHING) 327#define FRAME_INTR_EXIT \ 328 FRAME_INTR_XEXIT(rfi, SRR) 329#define FRAME_CRIT_ENTER(exc) \ 330 FRAME_INTR_XENTER(exc, 24, FRAME_INTR_SP, SAVE_SPRG1) 331#define FRAME_WDOG_ENTER(exc, sym) \ 332 FRAME_INTR_XENTER(exc, 24, FRAME_INTR_SP_NEW(sym), SAVE_SPRG1) 333#define FRAME_CRIT_EXIT \ 334 RESTORE_SRR0(%r4); \ 335 RESTORE_SRR1(%r5); \ 336 RESTORE_SPRG1(%r6); \ 337 FRAME_INTR_XEXIT(rfci, CSRR) 338 339 .text 340 .p2align 4 341_C_LABEL(critical_input_vector): 342 /* MSR[ME] is unchanged, all others cleared */ 343 FRAME_CRIT_PROLOGUE /* save SP r26-31 CR LR XER */ 344 FRAME_CRIT_ENTER(EXC_CII) 345 bl _C_LABEL(intr_critintr) /* critintr(tf) */ 346 FRAME_CRIT_EXIT 347 348 .p2align 4 349_C_LABEL(machine_check_vector): 350 /* all MSR bits are cleared */ 351 FRAME_MCHK_PROLOGUE /* save SP r25-31 CR LR XER */ 352 FRAME_MCHK_ENTER(EXC_MCHK) 353 /* 354 * MCAR/MCSR don't need to be saved early since MSR[ME] is cleared 355 * on entry. 356 */ 357 mfspr %r7, SPR_MCAR 358 mfspr %r6, SPR_MCSR 359 stw %r6, FRAME_MCSR(%r1) 360 stw %r7, FRAME_MCAR(%r1) 361 li %r3, T_MACHINE_CHECK 362 bl _C_LABEL(trap) /* trap(T_MACHINE_CHECK, tf) */ 363 FRAME_MCHK_EXIT 364 365 .p2align 4 366_C_LABEL(data_storage_vector): 367 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 368 FRAME_PROLOGUE_DEAR_ESR /* save r2 DEAR ESR r24-31 CR XER SRR */ 369 FRAME_ENTER_DEAR_ESR(EXC_DSI, %r4) 370 li %r3, T_DSI 371 /* FRAME_ENTER leaves SRR1 in %r31 */ 372trapenter: 373trapagain: 374 wrtee %r31 /* restore MSR[EE] */ 375 376 bl _C_LABEL(trap) /* trap(trapcode, tf) */ 377_C_LABEL(trapexit): 378 wrteei 0 /* disable interrupts */ 379# andis. %r0, %r31, PSL_CE@h 380# tweqi %r0, 0 381 andi. %r4, %r31, PSL_PR /* lets look at PSL_PR */ 382 beq trapdone /* if clear, skip to exit */ 383 lwz %r4, L_MD_ASTPENDING(%r13) /* get ast pending */ 384 cmplwi %r4, 0 /* is there an ast pending */ 385 beq+ trapdone /* nope, proceed to exit */ 386 li %r6, EXC_AST /* yes. */ 387 stw %r6, FRAME_EXC(%r1) /* pretend this is an AST */ 388 addi %r4, %r1, FRAME_TF /* get address of trap frame */ 389 li %r3, T_AST 390 b trapagain /* and deal with it */ 391trapdone: 392 FRAME_EXIT 393 394 .p2align 4 395_C_LABEL(instruction_storage_vector): 396 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 397 FRAME_PROLOGUE_ESR /* save ESR r2 r25-31 CR XER SRR0/1 */ 398 FRAME_ENTER_ESR(EXC_ISI, %r4) 399 li %r3, T_ISI 400 b trapenter 401 402 .p2align 4 403_ENTRY(external_input_vector) 404 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 405 FRAME_INTR_PROLOGUE /* save SP r25-31 CR LR XER */ 406 FRAME_INTR_ENTER(EXC_EXI) 407 408 bl _C_LABEL(intr_extintr) 409_C_LABEL(intrcall): 410 GET_CPUINFO(%r6) /* get curcpu() */ 411 lwz %r5, FRAME_SRR1(%r1) /* get saved SRR1 */ 412# andis. %r0, %r5, PSL_CE@h 413# tweqi %r0, 0 414 andi. %r4, %r5, PSL_PR /* lets look at PSL_PR */ 415 beq intrexit /* if clear, skip to exit */ 416 lwz %r4, L_MD_ASTPENDING(%r13) /* get ast pending */ 417 cmplwi %r4, 0 /* is there an ast pending */ 418 beq+ intrexit /* nope, proceed to exit */ 419 stmw %r14, FRAME_R14(%r1) /* save rest of registers */ 420 FRAME_SAVE_SPEFSCR 421 mr %r31, %r5 /* needed for trapagain */ 422 li %r4, EXC_AST /* */ 423 stw %r4, FRAME_EXC(%r1) /* pretend this is an AST */ 424 addi %r4, %r1, FRAME_TF /* get address of trap frame */ 425 li %r3, T_AST 426 b trapagain /* and deal with it */ 427intrexit: 428 FRAME_INTR_EXIT 429 430 .p2align 4 431_C_LABEL(alignment_vector): 432 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 433 FRAME_PROLOGUE_DEAR_ESR /* save SP r25-31 CR LR XER */ 434 FRAME_ENTER_DEAR_ESR(EXC_ALI, %r4) 435 li %r3, T_ALIGNMENT 436 b trapenter 437 438 .p2align 4 439_C_LABEL(program_vector): 440 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 441 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 442 FRAME_ENTER_ESR(EXC_PGM, %r4) 443 li %r3, T_PROGRAM 444 b trapenter 445 446#ifdef SPR_IVOR7 447 .p2align 4 448_C_LABEL(fp_unavailable_vector): 449 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 450 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 451 FRAME_ENTER_ESR(EXC_FPU, %r4) 452 li %r3, T_FP_UNAVAILABLE 453 b trapenter 454#endif 455 456 .p2align 4 457_C_LABEL(system_call_vector): 458 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 459 FRAME_PROLOGUE /* save SP r26-31 CR LR XER */ 460 FRAME_ENTER(EXC_SC, %r3) 461 462 wrteei 1 /* enable interrupts */ 463 lwz %r7, L_PROC(%r13) /* get proc for lwp */ 464 lwz %r8, P_MD_SYSCALL(%r7) /* get syscall */ 465 mtlr %r8 /* need to call indirect */ 466 blrl /* syscall(tf) */ 467_C_LABEL(sctrapexit): 468 wrteei 0 /* disable interrupts */ 469 lwz %r4, L_MD_ASTPENDING(%r13) /* get ast pending */ 470 cmplwi %r4, 0 /* is there an ast pending */ 471 beq+ trapdone /* nope, proceed to exit */ 472 li %r0, EXC_AST /* yes. */ 473 stw %r0, FRAME_EXC(%r1) /* pretend this is an AST */ 474 addi %r4, %r1, FRAME_TF /* get address of trap frame */ 475 li %r3, T_AST 476 b trapenter /* and deal with it */ 477 478#ifdef SPR_IVOR9 479 .p2align 4 480_C_LABEL(ap_unavailable_vector): 481 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 482 FRAME_PROLOGUE /* save SP r25-31 CR LR XER */ 483 FRAME_ENTER(EXC_PGM, %r4) 484 li %r3, T_AP_UNAVAILABLE 485 b trapenter 486#endif 487 488 .p2align 4 489_C_LABEL(decrementer_vector): 490 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 491 FRAME_INTR_PROLOGUE /* save SP r25-31 CR LR XER */ 492 FRAME_INTR_ENTER(EXC_DECR) 493 494 bl _C_LABEL(intr_decrintr) 495 b intrexit 496 497 .p2align 4 498_C_LABEL(fixed_interval_timer_vector): 499 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 500 FRAME_PROLOGUE /* save SP r25-31 CR LR XER */ 501 FRAME_INTR_ENTER(EXC_FIT) 502 503 bl _C_LABEL(intr_fitintr) 504 b intrexit 505 506 .data 507 .lcomm wdogstk,4096 508 .text 509 .p2align 4 510_C_LABEL(watchdog_timer_vector): 511 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 512 FRAME_CRIT_PROLOGUE /* save SP r25-31 CR LR XER */ 513 FRAME_WDOG_ENTER(EXC_WDOG, wdogstk+4096-CALLFRAMELEN) 514#if 0 515 lis %r11,0xe0004641@ha 516 addi %r11,%r11,0xe0004641@l 517 stb %r11,-65(%r11) 518#endif 519 520 bl _C_LABEL(intr_wdogintr) 521 FRAME_CRIT_EXIT 522 523 .p2align 4 524_C_LABEL(data_tlb_error_vector): 525 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 526 FRAME_TLBPROLOGUE 527 /* 528 * Registers as this point: 529 * 530 * r2 = cpu_info 531 * r20 = scratch 532 * r21 = scratch 533 * r22 = scratch 534 * r23 = scratch 535 * r24 = DEAR 536 * r25 = ESR 537 * r26 = saved r2 538 * r27 = CR 539 * r28 = XER 540 * r29 = scratch 541 * r30 = SRR0 542 * r31 = SRR1 543 * 544 * Except for r29, these values must be retained. However we must 545 * be cognizant of nesting. There are two cases here, both related. 546 * 547 * We get a critical input or machine check exception and the kernel 548 * stack doesn't have a TLB entry so we take an exception. The other 549 * nesting path is some page used by the exception handler will cause 550 * a TLB data error. 551 * 552 * The second case (more probable) is that the PTE loading will fail 553 * so we will have to do a hard trap to resolve it. But in doing so 554 * we need to save a trapframe which could result in another DTLB 555 * fault. 556 * 557 * In all cases, the save area stack shall protect us. 558 */ 559 /* 560 * Attempt to update the TLB from the page table. 561 */ 562 mflr %r29 /* save LR */ 563 mr %r23, %r24 /* address of exception */ 564 rlwinm %r22, %r31, /* index into ci_pmap_segtab */\ 565 MSR_DS+PTR_SCALESHIFT+1, \ 566 31-PTR_SCALESHIFT, \ 567 31-PTR_SCALESHIFT /* move PSL_DS[27] to bit 29 */ 568 bl pte_load 569 mtlr %r29 /* restore LR */ 570 /* 571 * If we returned, pte load failed so let trap deal with it but 572 * has kept the contents of r24-r31 (expect r29) intact. 573 */ 574 FRAME_TLBENTER(EXC_DSI) 575 li %r3, T_DATA_TLB_ERROR 576 b trapenter 577 578 .p2align 4 579_C_LABEL(instruction_tlb_error_vector): 580 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 581 FRAME_TLBPROLOGUE 582 /* 583 * Attempt to update the TLB from the page table. 584 */ 585 mflr %r29 /* save LR */ 586 mr %r23, %r30 /* PC of exception */ 587 rlwinm %r22, %r31, /* index into ci_pmap_segtab */\ 588 MSR_IS+PTR_SCALESHIFT+1, \ 589 31-PTR_SCALESHIFT, \ 590 31-PTR_SCALESHIFT /* move PSL_IS[26] to bit 29 */ 591 bl pte_load 592 mtlr %r29 /* restore LR */ 593 /* 594 * If we returned, pte load failed so let trap deal with it but 595 * has kept the contents of r24-r31 (expect r29) intact. 596 */ 597 FRAME_TLBENTER(EXC_ISI) 598 li %r3, T_INSTRUCTION_TLB_ERROR 599 b trapenter 600 601 .p2align 4 602_C_LABEL(debug_vector): 603 FRAME_CRIT_PROLOGUE /* save SP r25-31 CR LR XER */ 604 FRAME_CRIT_ENTER(EXC_DEBUG) 605 mfspr %r6, SPR_DBSR 606 stw %r6, FRAME_ESR(%r1) 607 li %r3, T_DEBUG 608 bl _C_LABEL(trap) 609 FRAME_CRIT_EXIT 610 611 .p2align 4 612_C_LABEL(spv_unavailable_vector): 613 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 614 FRAME_ENTER_ESR(EXC_VEC, %r4) 615 li %r3, T_SPE_UNAVAILABLE 616 b trapenter 617 618 .p2align 4 619_C_LABEL(fpdata_vector): 620 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 621 FRAME_ENTER_ESR(EXC_FPA, %r4) 622 li %r3, T_EMBEDDED_FP_DATA 623 b trapenter 624 625 .p2align 4 626_C_LABEL(fpround_vector): 627 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 628 FRAME_ENTER_ESR(EXC_FPA, %r4) 629 li %r3, T_EMBEDDED_FP_ROUND 630 b trapenter 631 632 .p2align 4 633_C_LABEL(perfmon_vector): 634 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 635 FRAME_ENTER_ESR(EXC_PERF, %r4) 636 li %r3, T_EMBEDDED_PERF_MONITOR 637 b trapenter 638 639 .p2align 4 640pte_load: 641 /* 642 * r2 = scratch 643 * r20 = scratch 644 * r21 = scratch 645 * r22 = index into ci_pmap_{kern,user}_segtab 646 * r23 = faulting address 647 * The rest are for reference and aren't modifiable. If the load 648 * fails, they will be used by FRAME_TLBENTER to create the trapframe. 649 * r24 = DEAR 650 * r25 = ESR 651 * r26 = saved r2 652 * r27 = CR 653 * r28 = XER 654 * r29 = LR 655 * r30 = SRR0 656 * r31 = SRR1 657 */ 658 cmplwi %cr2, %r22, 0 /* remember address space */ 659 GET_CPUINFO(%r2) 660 addi %r22, %r22, CI_PMAP_SEGTAB /* index into segtab(s) */ 661 lwzx %r20, %r22, %r2 /* load kern/user L1 PT addr */ 662 cmplwi %r20, 0 /* is segtab null? */ 663 beqlr %cr0 /* yes, return to fallback to trap */ 664 665 rlwinm %r22, %r23, NSEGPG_SCALESHIFT + PTR_SCALESHIFT, \ 666 31-(NSEGPG_SCALESHIFT + PTR_SCALESHIFT - 1), \ 667 31-PTR_SCALESHIFT /* extract addr bits [0:9] to [20:29] */ 668 lwzx %r20, %r22, %r20 /* load address of page table page */ 669 cmplwi %r20, 0 /* is page null? */ 670 beqlr %cr0 /* yes, return to fallback to trap */ 671 672 rlwinm %r22, %r23, \ 673 NSEGPG_SCALESHIFT + NPTEPG_SCALESHIFT + PTE_SCALESHIFT, \ 674 31-(NPTEPG_SCALESHIFT + PTE_SCALESHIFT - 1), \ 675 31-PTE_SCALESHIFT /* extract addr bits [10:19] to [20:29] */ 676 lwzx %r20, %r22, %r20 /* load PTE from page table page */ 677 cmplwi %r20, 0 /* is there a valid PTE? */ 678 beqlr %cr0 /* no, return to fallback to trap */ 679 680#if (PTE_UNSYNCED << 1) != PTE_xX 681#error PTE_UNSYNCED definition error 682#endif 683#if (PTE_UNMODIFIED << 1) != PTE_xW 684#error PTE_UNMODIFIED definition error 685#endif 686 andi. %r22, %r20, (PTE_UNSYNCED|PTE_UNMODIFIED) 687 /* Does the PTE need to be changed? */ 688 rotlwi %r22, %r22, 1 /* if so, clear the right PTE bits */ 689 andc %r20, %r20, %r22 /* pte &= ~((pte & (PTE_UNSYNCED|PTE_UNMODIFIED)) << 1)*/ 690 691 /* 692 * r24-r32 = (no touch) 693 * r23 = scratch (was fault addr) 694 * r22 = scratch 695 * r21 = scratch 696 * r20 = pte 697 * cr2 = AS 0=eq/!0=ne 698 */ 699 700 /* 701 * This is all E500 specific. We should have a patchable branch 702 * to support other BookE (440) implementations. 703 */ 704e500_pte_load: 705 bne+ %cr2, 1f /* user access? MAS1 is ok. */ 706 mfspr %r22, SPR_MAS1 /* get MAS1 */ 707 lis %r21, MAS1_TID@h /* get TID mask */ 708 andc %r22, %r22, %r21 /* clear TID */ 709 mtspr SPR_MAS1, %r22 /* save MAS1 */ 7101: 711 andi. %r21, %r20, PTE_WIMGE_MASK /* extract WIMGE from PTE */ 712 cmplwi %r21, PTE_M /* if just PTE_M is set, */ 713 beq+ %cr0, 2f /* skip munging mas2 */ 714 mfspr %r22, SPR_MAS2 /* get MAS2 (updated by error) */ 715 clrrwi %r22, %r22, PTE_RWX_SHIFT /* clear WIMGE bits */ 716 or %r22, %r22, %r21 /* combine with MAS2 contents */ 717 mtspr SPR_MAS2, %r22 /* put back into MAS2 */ 7182: 719 /* 720 * r23 = fault addr 721 * r22 = scratch 722 * r21 = scratch 723 * r20 = pte 724 */ 725 726 /* 727 * In MAS3, the protection bits are in the low 6 bits: 728 * UX SX UW SW UR SR 729 * The User bits are 1 bit left of their Supervisor counterparts. 730 * Rotate the PTE protection bits left until they wrap around to become 731 * the least significant bits, where the Supervisor protection bits 732 * are located. Increase the rotate amount by 1 to place them where 733 * the User protection bits are located. We get that 1 by extracting 734 * the MAS1[TS] (set for User access) and moving it to bit 31 (LSB). 735 */ 736 mfspr %r21, SPR_MAS1 /* get MAS1 which has TS bit */ 737 extrwi %r21, %r21, 1, 31-MAS1_TS_SHIFT 738 /* extract MAS1_TS to LSB */ 739 clrrwi %r23, %r20, PAGE_SHIFT /* clear non-RPN bits from PTE */ 740 andi. %r20, %r20, PTE_RWX_MASK /* isolate protection bits */ 741 rotrwi %r20, %r20, PTE_RWX_SHIFT 742 andi. %r22, %r20, (MAS3_SW|MAS3_SR) /* user pages need to be R/W by kernel */ 743 rotlw %r20, %r20, %r21 /* rotate protection to correct loc */ 744 or %r20, %r20, %r22 /* combine system protection bits */ 745 or %r23, %r23, %r20 /* combine RPN and protection bits */ 746 mtspr SPR_MAS3, %r23 /* put into MAS3 */ 747 isync /* because ECORE500RM tells us too */ 748 tlbwe /* write the TLB entry */ 749 /* 750 * Increment a counter to show how many tlb misses we've handled here. 751 */ 752 lmw %r30, CI_EV_TLBMISS_SOFT(%r2) 753 addic %r31, %r31, 1 754 addze %r30, %r30 755 stmw %r30, CI_EV_TLBMISS_SOFT(%r2) 756 /* 757 * Cleanup and leave. We know any higher priority exception will 758 * save and restore SPRG1 and %r2 thereby preserving their values. 759 * 760 * r24 = DEAR (don't care) 761 * r25 = ESR (don't care) 762 * r26 = saved r2 763 * r27 = CR 764 * r28 = XER 765 * r29 = LR 766 * r30 = LSW of counter 767 * r31 = MSW of counter 768 */ 769 mtlr %r29 /* restore Link Register */ 770 mtxer %r28 /* restore XER */ 771 mtcr %r27 /* restore Condition Register */ 772 mtsprg1 %r26 /* save saved r2 across load multiple */ 773 mfsprg3 %r2 /* get end of save area */ 774 addi %r2,%r2,-4*(32-20) /* adjust save area down */ 775 lmw %r20,0(%r2) /* restore r20-r31 */ 776 mtsprg3 %r2 /* save new end of save area */ 777 mfsprg1 %r2 /* restore r2 */ 778 rfi 779 780 .p2align 4 781 .globl _C_LABEL(exception_init) 782_C_LABEL(exception_init): 783 lis %r6,_C_LABEL(critical_input_vector)@h 784 mtspr SPR_IVPR, %r6 785 786 ori %r5,%r6,_C_LABEL(critical_input_vector)@l 787 mtspr SPR_IVOR0, %r5 788 789 ori %r5,%r6,_C_LABEL(machine_check_vector)@l 790 mtspr SPR_IVOR1, %r5 791 792 ori %r5,%r6,_C_LABEL(data_storage_vector)@l 793 mtspr SPR_IVOR2, %r5 794 795 ori %r5,%r6,_C_LABEL(instruction_storage_vector)@l 796 mtspr SPR_IVOR3, %r5 797 798 ori %r5,%r6,_C_LABEL(external_input_vector)@l 799 mtspr SPR_IVOR4, %r5 800 801 ori %r5,%r6,_C_LABEL(alignment_vector)@l 802 mtspr SPR_IVOR5, %r5 803 804 ori %r5,%r6,_C_LABEL(program_vector)@l 805 mtspr SPR_IVOR6, %r5 806 807#ifdef SPR_IVOR7 808 ori %r5,%r6,_C_LABEL(fp_unavailable_vector)@l 809 mtspr SPR_IVOR7, %r5 810#endif 811 812 ori %r5,%r6,_C_LABEL(system_call_vector)@l 813 mtspr SPR_IVOR8, %r5 814 815#ifdef SPR_IVOR9 816 ori %r5,%r6,_C_LABEL(ap_unavailable_vector)@l 817 mtspr SPR_IVOR9, %r5 818#endif 819 820 ori %r5,%r6,_C_LABEL(decrementer_vector)@l 821 mtspr SPR_IVOR10, %r5 822 823 ori %r5,%r6,_C_LABEL(fixed_interval_timer_vector)@l 824 mtspr SPR_IVOR11, %r5 825 826 ori %r5,%r6,_C_LABEL(watchdog_timer_vector)@l 827 mtspr SPR_IVOR12, %r5 828 829 ori %r5,%r6,_C_LABEL(data_tlb_error_vector)@l 830 mtspr SPR_IVOR13, %r5 831 832 ori %r5,%r6,_C_LABEL(instruction_tlb_error_vector)@l 833 mtspr SPR_IVOR14, %r5 834 835 ori %r5,%r6,_C_LABEL(debug_vector)@l 836 mtspr SPR_IVOR15, %r5 837 838 ori %r5,%r6,_C_LABEL(spv_unavailable_vector)@l 839 mtspr SPR_IVOR32, %r5 840 841 ori %r5,%r6,_C_LABEL(fpdata_vector)@l 842 mtspr SPR_IVOR33, %r5 843 844 ori %r5,%r6,_C_LABEL(fpround_vector)@l 845 mtspr SPR_IVOR34, %r5 846 847 ori %r5,%r6,_C_LABEL(perfmon_vector)@l 848 mtspr SPR_IVOR35, %r5 849 850 mfpir %r5 /* get Process ID register */ 851 cmplwi %r5,0 852 bnelr /* return if non-0 (non-primary) */ 853 854 lis %r5,_C_LABEL(powerpc_intrsw)@ha 855 stw %r3,_C_LABEL(powerpc_intrsw)@l(%r5) 856 857 blr 858 859#ifdef notyet 860 .data 861 .lcomm ddbstk,4096 862 .text 863 864_ENTRY(cpu_Debugger) 865 mflr %r0 866 stw %r0, CFRAME_LR(%r1) 867 868 mfmsr %r3 869 wrteei 0 870 mr %r4,%r1 871 lis %r10,ddbstk@ha 872 addi %r10,%r10,ddbstk@l 873 sub %r5,%r1,%r10 874 cmplwi %r5,4096 875 blt %cr0, 1f 876 addi %r1,%r10,4096-CALLFRAMELEN 8771: 878 stwu %r4,-FRAMELEN(%r1) 879 stw %r4,FRAME_R1(%r1) 880 stmw %r13,FRAME_R13(%r1) 881 mr %r26,%r0 882 mfcr %r27 883 mfxer %r28 884 mfctr %r29 885 mr %r30,%r0 886 mr %r31,%r3 887 stmw %r26,FRAME_LR(%r1) 888 mr %r31,%r1 889 mr %r1,%r10 890 addi %r4,%r1,FRAME_TF 891 li %r3,EXC_PGM 892 stw %r3,FRAME_EXC(%r1) 893 li %r3,T_PROGRAM 894 bl _C_LABEL(trap) 895 lmw %r26,FRAME_LR(%r1) 896 mtlr %r26 897 mtcr %r27 898 mtxer %r28 899 mtctr %r29 900 mr %r0,%r31 901 lmw %r13,FRAME_R13(%r1) 902 lwz %r1,FRAME_R1(%r1) 903 wrtee %r0 904 blr 905#endif /* notyet */ 906