1 /*- 2 * Copyright (c) 2014 The NetBSD Foundation, Inc. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to The NetBSD Foundation 6 * by Matt Thomas of 3am Software Foundry. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 * POSSIBILITY OF SUCH DAMAGE. 28 */ 29 #define __INTR_PRIVATE 30 #include <sys/cdefs.h> 31 32 __KERNEL_RCSID(0, "$NetBSD: octeon_cpunode.c,v 1.19 2021/04/24 23:36:42 thorpej Exp $"); 33 34 #include "locators.h" 35 #include "cpunode.h" 36 #include "opt_multiprocessor.h" 37 #include "opt_ddb.h" 38 39 #include <sys/param.h> 40 #include <sys/atomic.h> 41 #include <sys/cpu.h> 42 #include <sys/device.h> 43 #include <sys/lwp.h> 44 #include <sys/reboot.h> 45 #include <sys/wdog.h> 46 47 #include <uvm/uvm.h> 48 49 #include <dev/sysmon/sysmonvar.h> 50 51 #include <mips/cache.h> 52 #include <mips/mips_opcode.h> 53 #include <mips/mips3_clock.h> 54 #include <mips/mips3_pte.h> 55 56 #include <mips/cavium/octeonvar.h> 57 #include <mips/cavium/dev/octeon_ciureg.h> 58 #include <mips/cavium/dev/octeon_corereg.h> 59 60 extern struct cpu_softc octeon_cpu_softc[]; 61 62 struct cpunode_attach_args { 63 const char *cnaa_name; 64 int cnaa_cpunum; 65 }; 66 67 struct cpunode_softc { 68 device_t sc_dev; 69 device_t sc_wdog_dev; 70 }; 71 72 static int cpunode_mainbus_match(device_t, cfdata_t, void *); 73 static void cpunode_mainbus_attach(device_t, device_t, void *); 74 75 static int cpu_cpunode_match(device_t, cfdata_t, void *); 76 static void cpu_cpunode_attach(device_t, device_t, void *); 77 78 CFATTACH_DECL_NEW(cpunode, sizeof(struct cpunode_softc), 79 cpunode_mainbus_match, cpunode_mainbus_attach, NULL, NULL); 80 81 CFATTACH_DECL_NEW(cpu_cpunode, 0, 82 cpu_cpunode_match, cpu_cpunode_attach, NULL, NULL); 83 84 #ifdef MULTIPROCESSOR 85 CTASSERT(MAXCPUS <= sizeof(uint64_t) * NBBY); 86 volatile uint64_t cpus_booted = __BIT(0); /* cpu0 is always booted */ 87 #endif 88 89 static void wdog_cpunode_poke(void *arg); 90 91 static int 92 cpunode_mainbus_print(void *aux, const char *pnp) 93 { 94 struct cpunode_attach_args * const cnaa = aux; 95 96 if (pnp) 97 aprint_normal("%s", pnp); 98 99 if (cnaa->cnaa_cpunum != CPUNODECF_CORE_DEFAULT) 100 aprint_normal(" core %d", cnaa->cnaa_cpunum); 101 102 return UNCONF; 103 } 104 105 int 106 cpunode_mainbus_match(device_t parent, cfdata_t cf, void *aux) 107 { 108 109 return 1; 110 } 111 112 void 113 cpunode_mainbus_attach(device_t parent, device_t self, void *aux) 114 { 115 struct cpunode_softc * const sc = device_private(self); 116 const uint64_t fuse = octeon_xkphys_read_8(CIU_FUSE); 117 int cpunum = 0; 118 119 sc->sc_dev = self; 120 121 aprint_naive(": %u core%s\n", popcount64(fuse), fuse == 1 ? "" : "s"); 122 aprint_normal(": %u core%s", popcount64(fuse), fuse == 1 ? "" : "s"); 123 124 const uint64_t cvmctl = mips_cp0_cvmctl_read(); 125 aprint_normal(", %scrypto", (cvmctl & CP0_CVMCTL_NOCRYPTO) ? "no " : ""); 126 aprint_normal((cvmctl & CP0_CVMCTL_KASUMI) ? "+kasumi" : ""); 127 aprint_normal(", %s64bit-mul", (cvmctl & CP0_CVMCTL_NOMUL) ? "no " : ""); 128 if (cvmctl & CP0_CVMCTL_REPUN) 129 aprint_normal(", unaligned-access ok"); 130 #ifdef MULTIPROCESSOR 131 aprint_normal(", booted %#" PRIx64, cpus_booted); 132 #endif 133 aprint_normal("\n"); 134 135 for (uint64_t f = fuse; f != 0; f >>= 1, cpunum++) { 136 struct cpunode_attach_args cnaa = { 137 .cnaa_name = "cpu", 138 .cnaa_cpunum = cpunum, 139 }; 140 config_found(self, &cnaa, cpunode_mainbus_print, CFARG_EOL); 141 } 142 #if NWDOG > 0 143 struct cpunode_attach_args cnaa = { 144 .cnaa_name = "wdog", 145 .cnaa_cpunum = CPUNODECF_CORE_DEFAULT, 146 }; 147 config_found(self, &cnaa, cpunode_mainbus_print, CFARG_EOL); 148 #endif 149 } 150 151 int 152 cpu_cpunode_match(device_t parent, cfdata_t cf, void *aux) 153 { 154 struct cpunode_attach_args * const cnaa = aux; 155 const int cpunum = cf->cf_loc[CPUNODECF_CORE]; 156 157 return strcmp(cnaa->cnaa_name, cf->cf_name) == 0 158 && (cpunum == CPUNODECF_CORE_DEFAULT || cpunum == cnaa->cnaa_cpunum); 159 } 160 161 #if defined(MULTIPROCESSOR) 162 static bool 163 octeon_fixup_cpu_info_references(int32_t load_addr, uint32_t new_insns[2], 164 void *arg) 165 { 166 struct cpu_info * const ci = arg; 167 168 atomic_or_ulong(&curcpu()->ci_flags, CPUF_PRESENT); 169 170 KASSERT(MIPS_KSEG0_P(load_addr)); 171 #ifdef MULTIPROCESSOR 172 KASSERT(!CPU_IS_PRIMARY(curcpu())); 173 #endif 174 load_addr += (intptr_t)ci - (intptr_t)&cpu_info_store; 175 176 KASSERT((intptr_t)ci <= load_addr); 177 KASSERT(load_addr < (intptr_t)(ci + 1)); 178 179 KASSERT(INSN_LUI_P(new_insns[0])); 180 KASSERT(INSN_LOAD_P(new_insns[1]) || INSN_STORE_P(new_insns[1])); 181 182 /* 183 * Use the lui and load/store instruction as a prototype and 184 * make it refer to cpu1_info_store instead of cpu_info_store. 185 */ 186 new_insns[0] &= __BITS(31,16); 187 new_insns[1] &= __BITS(31,16); 188 new_insns[0] |= (uint16_t)((load_addr + 0x8000) >> 16); 189 new_insns[1] |= (uint16_t)load_addr; 190 #ifdef DEBUG_VERBOSE 191 printf("%s: %08x: insn#1 %08x: lui r%u, %d\n", 192 __func__, load_addr, new_insns[0], 193 (new_insns[0] >> 16) & 31, 194 (int16_t)new_insns[0]); 195 printf("%s: %08x: insn#2 %08x: %c%c r%u, %d(r%u)\n", 196 __func__, load_addr, new_insns[1], 197 INSN_LOAD_P(new_insns[1]) ? 'l' : 's', 198 INSN_LW_P(new_insns[1]) ? 'w' : 'd', 199 (new_insns[1] >> 16) & 31, 200 (int16_t)new_insns[1], 201 (new_insns[1] >> 21) & 31); 202 #endif 203 return true; 204 } 205 206 static void 207 octeon_cpu_init(struct cpu_info *ci) 208 { 209 extern const mips_locore_jumpvec_t mips64r2_locore_vec; 210 bool ok __diagused; 211 212 mips3_cp0_pg_mask_write(MIPS3_PG_SIZE_TO_MASK(PAGE_SIZE)); 213 mips3_cp0_wired_write(0); 214 (*mips64r2_locore_vec.ljv_tlb_invalidate_all)(); 215 mips3_cp0_wired_write(pmap_tlb0_info.ti_wired); 216 217 // First thing is setup the execption vectors for this cpu. 218 mips64r2_vector_init(&mips_splsw); 219 220 // Next rewrite those exceptions to use this cpu's cpu_info. 221 ok = mips_fixup_exceptions(octeon_fixup_cpu_info_references, ci); 222 KASSERT(ok); 223 224 (void) splhigh(); // make sure interrupts are masked 225 226 KASSERT((mipsNN_cp0_ebase_read() & MIPS_EBASE_CPUNUM) == ci->ci_cpuid); 227 KASSERT(curcpu() == ci); 228 KASSERT(ci->ci_cpl == IPL_HIGH); 229 KASSERT((mips_cp0_status_read() & MIPS_INT_MASK) == 0); 230 } 231 232 static void 233 octeon_cpu_run(struct cpu_info *ci) 234 { 235 236 octeon_intr_init(ci); 237 238 mips3_initclocks(); 239 KASSERTMSG(ci->ci_cpl == IPL_NONE, "cpl %d", ci->ci_cpl); 240 KASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE); 241 242 aprint_normal("%s: ", device_xname(ci->ci_dev)); 243 cpu_identify(ci->ci_dev); 244 } 245 #endif /* MULTIPROCESSOR */ 246 247 static void 248 cpu_cpunode_attach_common(device_t self, struct cpu_info *ci) 249 { 250 struct cpu_softc * const cpu __diagused = ci->ci_softc; 251 252 ci->ci_dev = self; 253 self->dv_private = ci; 254 255 KASSERTMSG(cpu != NULL, "ci %p index %d", ci, cpu_index(ci)); 256 257 #if NWDOG > 0 || defined(DDB) 258 /* XXXXXX __mips_n32 and MIPS_PHYS_TO_XKPHYS_CACHED needed here?????? */ 259 void **nmi_vector = (void *)MIPS_PHYS_TO_KSEG0(0x800 + 32*ci->ci_cpuid); 260 *nmi_vector = octeon_reset_vector; 261 262 struct vm_page * const pg = PMAP_ALLOC_POOLPAGE(UVM_PGA_ZERO); 263 KASSERT(pg != NULL); 264 const vaddr_t kva = PMAP_MAP_POOLPAGE(VM_PAGE_TO_PHYS(pg)); 265 KASSERT(kva != 0); 266 ci->ci_nmi_stack = (void *)(kva + PAGE_SIZE - sizeof(struct kernframe)); 267 #endif 268 269 #if NWDOG > 0 270 cpu->cpu_wdog_sih = softint_establish(SOFTINT_CLOCK|SOFTINT_MPSAFE, 271 wdog_cpunode_poke, cpu); 272 KASSERT(cpu->cpu_wdog_sih != NULL); 273 #endif 274 275 aprint_normal(": %lu.%02luMHz\n", 276 (ci->ci_cpu_freq + 5000) / 1000000, 277 ((ci->ci_cpu_freq + 5000) % 1000000) / 10000); 278 aprint_debug_dev(self, "hz cycles = %lu, delay divisor = %lu\n", 279 ci->ci_cycles_per_hz, ci->ci_divisor_delay); 280 281 if (CPU_IS_PRIMARY(ci)) { 282 aprint_normal("%s: ", device_xname(self)); 283 cpu_identify(self); 284 } 285 cpu_attach_common(self, ci); 286 #ifdef MULTIPROCESSOR 287 KASSERT(cpuid_infos[ci->ci_cpuid] == ci); 288 #endif 289 } 290 291 void 292 cpu_cpunode_attach(device_t parent, device_t self, void *aux) 293 { 294 struct cpunode_attach_args * const cnaa = aux; 295 const int cpunum = cnaa->cnaa_cpunum; 296 297 if (cpunum == 0) { 298 cpu_cpunode_attach_common(self, curcpu()); 299 #ifdef MULTIPROCESSOR 300 mips_locoresw.lsw_cpu_init = octeon_cpu_init; 301 mips_locoresw.lsw_cpu_run = octeon_cpu_run; 302 #endif 303 return; 304 } 305 #ifdef MULTIPROCESSOR 306 if ((boothowto & RB_MD1) != 0) { 307 aprint_naive("\n"); 308 aprint_normal(": multiprocessor boot disabled\n"); 309 return; 310 } 311 312 if (!(cpus_booted & __BIT(cpunum))) { 313 aprint_naive(" disabled\n"); 314 aprint_normal(" disabled (unresponsive)\n"); 315 return; 316 } 317 struct cpu_info * const ci = cpu_info_alloc(NULL, cpunum, 0, cpunum, 0); 318 319 ci->ci_softc = &octeon_cpu_softc[cpunum]; 320 ci->ci_softc->cpu_ci = ci; 321 322 cpu_cpunode_attach_common(self, ci); 323 324 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 325 for (int i = 0; i < 100 && !kcpuset_isset(cpus_hatched, cpunum); i++) { 326 delay(10000); 327 } 328 if (!kcpuset_isset(cpus_hatched, cpunum)) { 329 #ifdef DDB 330 aprint_verbose_dev(self, "hatch failed ci=%p flags=%#lx\n", ci, ci->ci_flags); 331 cpu_Debugger(); 332 #endif 333 panic("%s failed to hatch: ci=%p flags=%#lx", 334 cpu_name(ci), ci, ci->ci_flags); 335 } 336 #else 337 aprint_naive(": disabled\n"); 338 aprint_normal(": disabled (uniprocessor kernel)\n"); 339 #endif 340 } 341 342 #if NWDOG > 0 343 struct wdog_softc { 344 struct sysmon_wdog sc_smw; 345 device_t sc_dev; 346 u_int sc_wdog_period; 347 bool sc_wdog_armed; 348 }; 349 350 #ifndef OCTEON_WDOG_PERIOD_DEFAULT 351 #define OCTEON_WDOG_PERIOD_DEFAULT 4 352 #endif 353 354 static int wdog_cpunode_match(device_t, cfdata_t, void *); 355 static void wdog_cpunode_attach(device_t, device_t, void *); 356 357 CFATTACH_DECL_NEW(wdog_cpunode, sizeof(struct wdog_softc), 358 wdog_cpunode_match, wdog_cpunode_attach, NULL, NULL); 359 360 static int 361 wdog_cpunode_setmode(struct sysmon_wdog *smw) 362 { 363 struct wdog_softc * const sc = smw->smw_cookie; 364 365 if ((smw->smw_mode & WDOG_MODE_MASK) == WDOG_MODE_DISARMED) { 366 if (sc->sc_wdog_armed) { 367 CPU_INFO_ITERATOR cii; 368 struct cpu_info *ci; 369 for (CPU_INFO_FOREACH(cii, ci)) { 370 struct cpu_softc * const cpu = ci->ci_softc; 371 uint64_t wdog = mips3_ld(cpu->cpu_wdog); 372 wdog &= ~CIU_WDOGX_MODE; 373 mips3_sd(cpu->cpu_pp_poke, wdog); 374 aprint_verbose_dev(sc->sc_dev, 375 "%s: disable wdog=%#"PRIx64"\n", 376 cpu_name(ci), wdog); 377 mips3_sd(cpu->cpu_wdog, wdog); 378 mips3_sd(cpu->cpu_pp_poke, wdog); 379 } 380 sc->sc_wdog_armed = false; 381 } 382 } else if (!sc->sc_wdog_armed) { 383 kpreempt_disable(); 384 struct cpu_info *ci = curcpu(); 385 if (smw->smw_period == WDOG_PERIOD_DEFAULT) { 386 smw->smw_period = OCTEON_WDOG_PERIOD_DEFAULT; 387 } 388 uint64_t wdog_len = smw->smw_period * ci->ci_cpu_freq; 389 // 390 // This wdog is a 24-bit counter that decrements every 256 391 // cycles. This is then a 32-bit counter so as long wdog_len 392 // doesn't overflow a 32-bit value, we are fine. We write the 393 // 16-bits of the 32-bit period. 394 if ((wdog_len >> 32) != 0) { 395 kpreempt_enable(); 396 return EINVAL; 397 } 398 sc->sc_wdog_period = smw->smw_period; 399 CPU_INFO_ITERATOR cii; 400 for (CPU_INFO_FOREACH(cii, ci)) { 401 struct cpu_softc * const cpu = ci->ci_softc; 402 uint64_t wdog = mips3_ld(cpu->cpu_wdog); 403 wdog &= ~(CIU_WDOGX_MODE|CIU_WDOGX_LEN); 404 wdog |= __SHIFTIN(3, CIU_WDOGX_MODE); 405 wdog |= __SHIFTIN(wdog_len >> 16, CIU_WDOGX_LEN); 406 aprint_verbose_dev(sc->sc_dev, 407 "%s: enable wdog=%#"PRIx64" (%#"PRIx64")\n", 408 cpu_name(ci), wdog, wdog_len); 409 mips3_sd(cpu->cpu_wdog, wdog); 410 } 411 sc->sc_wdog_armed = true; 412 kpreempt_enable(); 413 } 414 return 0; 415 } 416 417 static void 418 wdog_cpunode_poke(void *arg) 419 { 420 struct cpu_softc *cpu = arg; 421 422 mips3_sd(cpu->cpu_pp_poke, 0); 423 } 424 425 static int 426 wdog_cpunode_tickle(struct sysmon_wdog *smw) 427 { 428 429 wdog_cpunode_poke(curcpu()->ci_softc); 430 #ifdef MULTIPROCESSOR 431 // We need to send IPIs to the other CPUs to poke their wdog. 432 cpu_send_ipi(NULL, IPI_WDOG); 433 #endif 434 return 0; 435 } 436 437 int 438 wdog_cpunode_match(device_t parent, cfdata_t cf, void *aux) 439 { 440 struct cpunode_softc * const sc = device_private(parent); 441 struct cpunode_attach_args * const cnaa = aux; 442 const int cpunum = cf->cf_loc[CPUNODECF_CORE]; 443 444 return sc->sc_wdog_dev == NULL 445 && strcmp(cnaa->cnaa_name, cf->cf_name) == 0 446 && cpunum == CPUNODECF_CORE_DEFAULT; 447 } 448 449 void 450 wdog_cpunode_attach(device_t parent, device_t self, void *aux) 451 { 452 struct cpunode_softc * const psc = device_private(parent); 453 struct wdog_softc * const sc = device_private(self); 454 cfdata_t const cf = device_cfdata(self); 455 456 psc->sc_wdog_dev = self; 457 458 sc->sc_dev = self; 459 sc->sc_smw.smw_name = device_xname(self); 460 sc->sc_smw.smw_cookie = sc; 461 sc->sc_smw.smw_setmode = wdog_cpunode_setmode; 462 sc->sc_smw.smw_tickle = wdog_cpunode_tickle; 463 sc->sc_smw.smw_period = OCTEON_WDOG_PERIOD_DEFAULT; 464 sc->sc_wdog_period = sc->sc_smw.smw_period; 465 466 /* 467 * We need one softint per cpu. It's to tickle the softints on 468 * other CPUs. 469 */ 470 #if 0 /* XXX unused? */ 471 CPU_INFO_ITERATOR cii; 472 struct cpu_info *ci; 473 for (CPU_INFO_FOREACH(cii, ci)) { 474 } 475 #endif 476 477 aprint_normal(": default period is %u second%s\n", 478 sc->sc_wdog_period, sc->sc_wdog_period == 1 ? "" : "s"); 479 480 if (sysmon_wdog_register(&sc->sc_smw) != 0) { 481 aprint_error_dev(self, "unable to register with sysmon\n"); 482 return; 483 } 484 485 if (cf->cf_flags & 1) { 486 int error = sysmon_wdog_setmode(&sc->sc_smw, WDOG_MODE_KTICKLE, 487 sc->sc_wdog_period); 488 if (error) 489 aprint_error_dev(self, 490 "failed to start kernel tickler: %d\n", error); 491 } 492 } 493 #endif /* NWDOG > 0 */ 494