1/* $OpenBSD: locore.S,v 1.48 2023/12/12 07:37:21 deraadt Exp $ */ 2 3/* 4 * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19#include "assym.h" 20 21#include <sys/errno.h> 22#include <sys/syscall.h> 23 24#include <machine/param.h> 25#include <machine/opal.h> 26#include <machine/psl.h> 27#include <machine/asm.h> 28 29 .abiversion 2 30 31 .rodata 32 33 .globl sigcode 34sigcode: 35 addi %r1, %r1, -32 36 mtctr %r12 37 bctrl 38 addi %r3, %r1, 32+SF_SC 39 li %r0, SYS_sigreturn 40 .globl sigcodecall 41sigcodecall: 42 sc 43 .globl sigcoderet 44sigcoderet: 45 .globl esigcode 46esigcode: 47 /* FALLTHROUGH */ 48 .globl sigfill 49sigfill: 50 .long 0 51esigfill: 52 .globl sigfillsiz 53sigfillsiz: 54 .long esigfill - sigfill 55 56 .text 57 58#ifdef MULTIPROCESSOR 59 60 .globl cpu_hatch 61cpu_hatch: 62 bl 1f 631: mflr %r31 64 addis %r2, %r31, (.TOC. - 1b)@ha 65 addi %r2, %r2, (.TOC. - 1b)@l 66 67 addis %r1, %r2, tmpstack_end@toc@ha 68 addi %r1, %r1, tmpstack_end@toc@l 69 addi %r1, %r1, -32 70 71 bl cpu_bootstrap 72 73 mfsprg0 %r31 74 ld %r1, CI_INITSTACK_END(%r31) 75 addi %r1, %r1, -32 76 77 bl cpu_start_secondary 78 b . 79 80 .data 81 82 .align PAGE_SHIFT 83tmpstack: 84 .space PAGE_SIZE 85tmpstack_end: 86 87 .text 88 89/* For Power ISA v3, stop a hardware thread forever. */ 90 .globl cpu_hatch_and_stop 91cpu_hatch_and_stop: 92 bl 1f 931: mflr %r2 94 addis %r2, %r2, (.TOC. - 1b)@ha 95 addi %r2, %r2, (.TOC. - 1b)@l 96 /* Clear PECE bits to disable exiting from idle. */ 97 li %r3, 0 98 mtspr 318, %r3 /* lpcr = 0 */ 99 /* Set psscr to request idle state. */ 100 addis %r3, %r2, cpu_idle_state_psscr@toc@ha 101 ld %r3, cpu_idle_state_psscr@toc@l(%r3) 102 mtspr 823, %r3 1031: stop 104 b 1b /* Lighter states might not use PECE. */ 105 106#endif 107 108 .globl cpu_idle_spin 109cpu_idle_spin: 110 blr 111 112/* Idle for Power ISA v3. */ 113 .globl cpu_idle_stop 114cpu_idle_stop: 115 /* Set psscr to request idle state. */ 116 addis %r3, %r2, cpu_idle_state_psscr@toc@ha 117 ld %r3, cpu_idle_state_psscr@toc@l(%r3) 118 mtspr 823, %r3 119 /* 120 * POWER9 23.5.9.2 State Loss and Restoration: We may lose 121 * "any nonhypervisor thread context (such as, GPRs, VSRs, 122 * FPRs)" and "the following SPRs: CR, FPSCR, VSCR, XER, DSCR, 123 * AMR, IAMR, UAMOR, AMOR, DAWR, DAWRX." 124 */ 125 mflr %r3 126 mfcr %r4 127 std %r3, 16(%r1) 128 stw %r4, 8(%r1) 129 std %r31, -8(%r1) 130 std %r30, -16(%r1) 131 std %r29, -24(%r1) 132 std %r28, -32(%r1) 133 std %r27, -40(%r1) 134 std %r26, -48(%r1) 135 std %r25, -56(%r1) 136 std %r24, -64(%r1) 137 std %r23, -72(%r1) 138 std %r22, -80(%r1) 139 std %r21, -88(%r1) 140 std %r20, -96(%r1) 141 std %r19, -104(%r1) 142 std %r18, -112(%r1) 143 std %r17, -120(%r1) 144 std %r16, -128(%r1) 145 std %r15, -136(%r1) 146 std %r14, -144(%r1) 147 /* Red zone ends at -288(%r1). */ 148 mfsprg0 %r3 149 std %r1, CI_IDLE_SP_SAVE(%r3) 150 stop 151 /* If we continue here, then we lost no context. */ 152 blr 153 154/* Come here from the system reset vector (rsttrapcode). */ 155 .globl cpu_idle_restore_context 156cpu_idle_restore_context: 157 bl 1f 1581: mflr %r2 159 addis %r2, %r2, (.TOC. - 1b)@ha 160 addi %r2, %r2, (.TOC. - 1b)@l /* TOC pointer */ 161 mfsprg0 %r3 162 ld %r1, CI_IDLE_SP_SAVE(%r3) /* stack pointer */ 163 mfmsr %r4 164 ori %r4, %r4, PSL_DR@l /* data relocation on */ 165 mtmsr %r4 166 ld %r14, -144(%r1) 167 ld %r15, -136(%r1) 168 ld %r16, -128(%r1) 169 ld %r17, -120(%r1) 170 ld %r18, -112(%r1) 171 ld %r19, -104(%r1) 172 ld %r20, -96(%r1) 173 ld %r21, -88(%r1) 174 ld %r22, -80(%r1) 175 ld %r23, -72(%r1) 176 ld %r24, -64(%r1) 177 ld %r25, -56(%r1) 178 ld %r26, -48(%r1) 179 ld %r27, -40(%r1) 180 ld %r28, -32(%r1) 181 ld %r29, -24(%r1) 182 ld %r30, -16(%r1) 183 ld %r31, -8(%r1) 184 lwz %r4, 8(%r1) /* cr */ 185 ld %r5, 16(%r1) /* lr */ 186 mtcr %r4 187 mtsrr0 %r5 188 rfid /* return from system reset interrupt */ 189 190 .globl cpu_switchto_asm 191cpu_switchto_asm: 192 cmpdi %r3, 0 193 beq 1f 194 195 mflr %r0 196 std %r0, 16(%r1) 197 mfcr %r10 198 stdu %r1, -SFRAMELEN(%r1) 199 std %r10, SF_CR(%r1) 200 std %r14, SF_R14(%r1) 201 std %r15, SF_R15(%r1) 202 std %r16, SF_R16(%r1) 203 std %r17, SF_R17(%r1) 204 std %r18, SF_R18(%r1) 205 std %r19, SF_R19(%r1) 206 std %r20, SF_R20(%r1) 207 std %r21, SF_R21(%r1) 208 std %r22, SF_R22(%r1) 209 std %r23, SF_R23(%r1) 210 std %r24, SF_R24(%r1) 211 std %r25, SF_R25(%r1) 212 std %r26, SF_R26(%r1) 213 std %r27, SF_R27(%r1) 214 std %r28, SF_R28(%r1) 215 std %r29, SF_R29(%r1) 216 std %r30, SF_R30(%r1) 217 std %r31, SF_R31(%r1) 218 219 ld %r31, P_ADDR(%r3) 220 std %r1, PCB_SP(%r31) 221 2221: 223 mfsprg0 %r30 224 225 li %r31, SONPROC 226 stb %r31, P_STAT(%r4) 227 std %r4, CI_CURPROC(%r30) 228 229 ld %r31, P_ADDR(%r4) 230 std %r31, CI_CURPCB(%r30) 231 ld %r1, PCB_SP(%r31) 232#if PCB_SP != 0 233 addi %r31, %r31, PCB_SP 234#endif 235 stdcx. %r1, 0, %r31 /* clear a possible reservation */ 236 ld %r0, (SFRAMELEN + 16)(%r1) 237 mtlr %r0 238 RETGUARD_SETUP(cpu_switchto_asm, %r11) 239 240 ld %r31, P_MD_USER_SLB_PA(%r4) 241 std %r31, CI_USER_SLB_PA(%r30) 242 243 ld %r31, SF_R31(%r1) 244 ld %r30, SF_R30(%r1) 245 ld %r29, SF_R29(%r1) 246 ld %r28, SF_R28(%r1) 247 ld %r27, SF_R27(%r1) 248 ld %r26, SF_R26(%r1) 249 ld %r25, SF_R25(%r1) 250 ld %r24, SF_R24(%r1) 251 ld %r23, SF_R23(%r1) 252 ld %r22, SF_R22(%r1) 253 ld %r21, SF_R21(%r1) 254 ld %r20, SF_R20(%r1) 255 ld %r19, SF_R19(%r1) 256 ld %r18, SF_R18(%r1) 257 ld %r17, SF_R17(%r1) 258 ld %r16, SF_R16(%r1) 259 ld %r15, SF_R15(%r1) 260 ld %r14, SF_R14(%r1) 261 ld %r10, SF_CR(%r1) 262 addi %r1, %r1, SFRAMELEN 263 mtcr %r10 264 RETGUARD_CHECK(cpu_switchto_asm, %r11) 265 blr 266 267 .globl kcopy 268kcopy: 269 RETGUARD_SETUP(kcopy, %r11) 270 mfsprg0 %r7 271 ld %r7, CI_CURPCB(%r7) 272 ld %r8, PCB_ONFAULT(%r7) 273 addis %r9, %r2, .Lkcopyfault@toc@ha 274 addi %r9, %r9, .Lkcopyfault@toc@l 275 std %r9, PCB_ONFAULT(%r7) 276 277 rldicl. %r6, %r5, 61, 3 278 beq 0, .Lkcopywords 279 280 addi %r3, %r3, -8 281 addi %r4, %r4, -8 282 mtctr %r6 2831: 284 ldu %r6, 8(%r3) 285 stdu %r6, 8(%r4) 286 bdnz 1b 287 288 addi %r3, %r3, 8 289 addi %r4, %r4, 8 290 291.Lkcopywords: 292 andi. %r6, %r5, 4 293 beq 0, .Lkcopybytes 294 295 lwz %r6, 0(%r3) 296 stw %r6, 0(%r4) 297 298 addi %r3, %r3, 4 299 addi %r4, %r4, 4 300 301.Lkcopybytes: 302 andi. %r6, %r5, 3 303 beq 0, .Lkcopysuccess 304 305 addi %r3, %r3, -1 306 addi %r4, %r4, -1 307 mtctr %r6 3082: 309 lbzu %r6, 1(%r3) 310 stbu %r6, 1(%r4) 311 bdnz 2b 312 313.Lkcopysuccess: 314 std %r8, PCB_ONFAULT(%r7) 315 li %r3, 0 316.Lkcopydone: 317 RETGUARD_CHECK(kcopy, %r11) 318 blr 319 320.Lkcopyfault: 321 std %r8, PCB_ONFAULT(%r7) 322 li %r3, EFAULT 323 b .Lkcopydone 324 325 .globl copystr 326copystr: 327 RETGUARD_SETUP(copystr, %r11) 328 mfsprg0 %r7 329 ld %r7, CI_CURPCB(%r7) 330 ld %r8, PCB_ONFAULT(%r7) 331 addis %r9, %r2, .Lcopystrfault@toc@ha 332 addi %r9, %r9, .Lcopystrfault@toc@l 333 std %r9, PCB_ONFAULT(%r7) 334 335 cmpdi %r5, 0 336 beq .Lcopystrtoolong 337 338 addi %r3, %r3, -1 339 addi %r4, %r4, -1 340 mtctr %r5 341 li %r9, 0 3421: 343 addi %r9, %r9, 1 344 lbzu %r5, 1(%r3) 345 stbu %r5, 1(%r4) 346 cmplwi %r5, 0 347 beq .Lcopystrsuccess 348 bdnz 1b 349 350.Lcopystrtoolong: 351 li %r3, ENAMETOOLONG 352 b .Lcopystrcleanup 353 354.Lcopystrfault: 355 li %r3, EFAULT 356 b .Lcopystrcleanup 357 358.Lcopystrsuccess: 359 li %r3, 0 360 361.Lcopystrcleanup: 362 cmpldi %r6, 0 363 beq 2f 364 std %r9, 0(%r6) 3652: 366 std %r8, PCB_ONFAULT(%r7) 367 RETGUARD_CHECK(copystr, %r11) 368 blr 369 370 .globl proc_trampoline 371 .type proc_trampoline, @function 372proc_trampoline: 373 bl proc_trampoline_mi 374 mr %r12, %r31 375 mr %r3, %r30 376 mtctr %r12 377 bctrl 378 b trapexit 379 380/* 381 * OPAL interfaces 382 */ 383 384 .data 385 386 .globl opal_base 387opal_base: 388 .quad 0 389 .globl opal_entry 390opal_entry: 391 .quad 0 392 393 .text 394 395#define OPAL_CALL(opal_token, name) \ 396 .globl name; \ 397name: \ 398 RETGUARD_SETUP(opal_call, %r12); \ 399 li %r0, opal_token; \ 400 b opal_call 401 402OPAL_CALL(OPAL_TEST, opal_test) 403OPAL_CALL(OPAL_CONSOLE_WRITE, opal_console_write) 404OPAL_CALL(OPAL_CONSOLE_READ, opal_console_read) 405OPAL_CALL(OPAL_RTC_READ, opal_rtc_read) 406OPAL_CALL(OPAL_RTC_WRITE, opal_rtc_write) 407OPAL_CALL(OPAL_CEC_POWER_DOWN, opal_cec_power_down) 408OPAL_CALL(OPAL_CEC_REBOOT, opal_cec_reboot) 409OPAL_CALL(OPAL_HANDLE_INTERRUPT, opal_handle_interrupt) 410OPAL_CALL(OPAL_POLL_EVENTS, opal_poll_events) 411OPAL_CALL(OPAL_PCI_CONFIG_READ_WORD, opal_pci_config_read_word) 412OPAL_CALL(OPAL_PCI_CONFIG_WRITE_WORD, opal_pci_config_write_word) 413OPAL_CALL(OPAL_SET_XIVE, opal_set_xive) 414OPAL_CALL(OPAL_GET_XIVE, opal_get_xive) 415OPAL_CALL(OPAL_PCI_EEH_FREEZE_STATUS, opal_pci_eeh_freeze_status) 416OPAL_CALL(OPAL_PCI_EEH_FREEZE_CLEAR, opal_pci_eeh_freeze_clear) 417OPAL_CALL(OPAL_PCI_PHB_MMIO_ENABLE, opal_pci_phb_mmio_enable) 418OPAL_CALL(OPAL_PCI_SET_PHB_MEM_WINDOW, opal_pci_set_phb_mem_window) 419OPAL_CALL(OPAL_PCI_MAP_PE_MMIO_WINDOW, opal_pci_map_pe_mmio_window) 420OPAL_CALL(OPAL_PCI_SET_PE, opal_pci_set_pe) 421OPAL_CALL(OPAL_PCI_SET_XIVE_PE, opal_pci_set_xive_pe) 422OPAL_CALL(OPAL_GET_MSI_32, opal_get_msi_32) 423OPAL_CALL(OPAL_GET_MSI_64, opal_get_msi_64) 424OPAL_CALL(OPAL_START_CPU, opal_start_cpu) 425OPAL_CALL(OPAL_PCI_MAP_PE_DMA_WINDOW, opal_pci_map_pe_dma_window) 426OPAL_CALL(OPAL_PCI_MAP_PE_DMA_WINDOW_REAL, opal_pci_map_pe_dma_window_real) 427OPAL_CALL(OPAL_PCI_RESET, opal_pci_reset) 428OPAL_CALL(OPAL_REINIT_CPUS, opal_reinit_cpus) 429OPAL_CALL(OPAL_CHECK_TOKEN, opal_check_token) 430OPAL_CALL(OPAL_SENSOR_READ, opal_sensor_read) 431OPAL_CALL(OPAL_IPMI_SEND, opal_ipmi_send) 432OPAL_CALL(OPAL_IPMI_RECV, opal_ipmi_recv) 433OPAL_CALL(OPAL_CONSOLE_FLUSH, opal_console_flush) 434OPAL_CALL(OPAL_XIVE_RESET, opal_xive_reset) 435OPAL_CALL(OPAL_XIVE_GET_IRQ_INFO, opal_xive_get_irq_info) 436OPAL_CALL(OPAL_XIVE_GET_IRQ_CONFIG, opal_xive_get_irq_config) 437OPAL_CALL(OPAL_XIVE_SET_IRQ_CONFIG, opal_xive_set_irq_config) 438OPAL_CALL(OPAL_XIVE_GET_QUEUE_INFO, opal_xive_get_queue_info) 439OPAL_CALL(OPAL_XIVE_SET_QUEUE_INFO, opal_xive_set_queue_info) 440OPAL_CALL(OPAL_XIVE_GET_VP_INFO, opal_xive_get_vp_info) 441OPAL_CALL(OPAL_XIVE_SET_VP_INFO, opal_xive_set_vp_info) 442OPAL_CALL(OPAL_XIVE_DUMP, opal_xive_dump) 443OPAL_CALL(OPAL_SENSOR_READ_U64, opal_sensor_read_u64) 444 445opal_call: 446 mflr %r11 447 std %r11, 16(%r1) 448 stdu %r1, -40(%r1) 449 std %r2, 24(%r1) 450 RETGUARD_SAVE(%r12, 32(%r1)) 451 452 addis %r11, %r2, opal_base@toc@ha 453 addi %r11, %r11, opal_base@toc@l 454 addis %r12, %r2, opal_entry@toc@ha 455 addi %r12, %r12, opal_entry@toc@l 456 ld %r11, 0(%r11) 457 ld %r12, 0(%r12) 458 459 /* Save MSR */ 460 std %r31, 12(%r1) 461 mfmsr %r31 462 463 /* Disable translation and external interrupts */ 464 andi. %r2, %r31, ~(PSL_DR|PSL_IR|PSL_EE)@l 465 mtmsr %r2 466 isync 467 468 mr %r2, %r11 469 mtctr %r12 470 bctrl 471 472 /* Restore MSR */ 473 mtmsrd %r31 474 isync 475 ld %r31, 12(%r1) 476 477 RETGUARD_LOAD(%r11, 32(%r1)) 478 ld %r2, 24(%r1) 479 addi %r1, %r1, 40 480 ld %r0, 16(%r1) 481 mtlr %r0 482 RETGUARD_CHECK(opal_call, %r11) 483 blr 484 485/* 486 * DDB support code 487 */ 488 489#ifdef DDB 490 .globl db_enter 491db_enter: 492 trap 493 /* The trap handler will return from db_enter(). */ 494 495 .globl setjmp 496setjmp: 497 RETGUARD_SETUP(setjmp, %r11) 498 mflr %r0 499 mfcr %r5 500 std %r0, 0x00(%r3) /* lr */ 501 std %r5, 0x08(%r3) /* cr */ 502 std %r1, 0x10(%r3) 503 std %r2, 0x18(%r3) 504 std %r14, 0x20(%r3) 505 std %r15, 0x28(%r3) 506 std %r16, 0x30(%r3) 507 std %r17, 0x38(%r3) 508 std %r18, 0x40(%r3) 509 std %r19, 0x48(%r3) 510 std %r20, 0x50(%r3) 511 std %r21, 0x58(%r3) 512 std %r22, 0x60(%r3) 513 std %r23, 0x68(%r3) 514 std %r24, 0x70(%r3) 515 std %r25, 0x78(%r3) 516 std %r26, 0x80(%r3) 517 std %r27, 0x88(%r3) 518 std %r28, 0x90(%r3) 519 std %r29, 0x98(%r3) 520 std %r30, 0xa0(%r3) 521 std %r31, 0xa8(%r3) 522 li %r3, 0 /* return 0 */ 523 RETGUARD_CHECK(setjmp, %r11) 524 blr 525 526 .globl longjmp 527longjmp: 528 ld %r0, 0x00(%r3) /* lr */ 529 ld %r5, 0x08(%r3) /* cr */ 530 mtlr %r0 531 RETGUARD_SETUP(longjmp, %r11) 532 mtcr %r5 533 ld %r1, 0x10(%r3) 534 ld %r2, 0x18(%r3) 535 ld %r14, 0x20(%r3) 536 ld %r15, 0x28(%r3) 537 ld %r16, 0x30(%r3) 538 ld %r17, 0x38(%r3) 539 ld %r18, 0x40(%r3) 540 ld %r19, 0x48(%r3) 541 ld %r20, 0x50(%r3) 542 ld %r21, 0x58(%r3) 543 ld %r22, 0x60(%r3) 544 ld %r23, 0x68(%r3) 545 ld %r24, 0x70(%r3) 546 ld %r25, 0x78(%r3) 547 ld %r26, 0x80(%r3) 548 ld %r27, 0x88(%r3) 549 ld %r28, 0x90(%r3) 550 ld %r29, 0x98(%r3) 551 ld %r30, 0xa0(%r3) 552 ld %r31, 0xa8(%r3) 553 li %r3, 1 /* return non-zero */ 554 RETGUARD_CHECK(longjmp, %r11) 555 blr 556#endif 557