1 /*- 2 * Copyright (c) 2014 The NetBSD Foundation, Inc. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to The NetBSD Foundation 6 * by Matt Thomas of 3am Software Foundry. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 * POSSIBILITY OF SUCH DAMAGE. 28 */ 29 #define __INTR_PRIVATE 30 #include <sys/cdefs.h> 31 32 __KERNEL_RCSID(0, "$NetBSD"); 33 34 #include "locators.h" 35 #include "cpunode.h" 36 #include "opt_multiprocessor.h" 37 #include "opt_ddb.h" 38 39 #include <sys/param.h> 40 #include <sys/device.h> 41 #include <sys/lwp.h> 42 #include <sys/cpu.h> 43 #include <sys/atomic.h> 44 #include <sys/wdog.h> 45 46 #include <uvm/uvm.h> 47 48 #include <dev/sysmon/sysmonvar.h> 49 50 #include <mips/cache.h> 51 #include <mips/mips_opcode.h> 52 #include <mips/mips3_clock.h> 53 54 #include <mips/cavium/octeonvar.h> 55 #include <mips/cavium/dev/octeon_ciureg.h> 56 #include <mips/cavium/dev/octeon_corereg.h> 57 58 struct cpunode_attach_args { 59 const char *cnaa_name; 60 int cnaa_cpunum; 61 }; 62 63 struct cpunode_softc { 64 device_t sc_dev; 65 device_t sc_wdog_dev; 66 uint64_t sc_fuse; 67 }; 68 69 static int cpunode_mainbus_match(device_t, cfdata_t, void *); 70 static void cpunode_mainbus_attach(device_t, device_t, void *); 71 72 static int cpu_cpunode_match(device_t, cfdata_t, void *); 73 static void cpu_cpunode_attach(device_t, device_t, void *); 74 75 CFATTACH_DECL_NEW(cpunode, sizeof(struct cpunode_softc), 76 cpunode_mainbus_match, cpunode_mainbus_attach, NULL, NULL); 77 78 CFATTACH_DECL_NEW(cpu_cpunode, 0, 79 cpu_cpunode_match, cpu_cpunode_attach, NULL, NULL); 80 81 kcpuset_t *cpus_booted; 82 83 void octeon_reset_vector(void); 84 85 static int 86 cpunode_mainbus_print(void *aux, const char *pnp) 87 { 88 struct cpunode_attach_args * const cnaa = aux; 89 90 if (cnaa->cnaa_cpunum != CPUNODECF_CORE_DEFAULT) 91 aprint_normal(" core %d", cnaa->cnaa_cpunum); 92 93 return UNCONF; 94 } 95 96 int 97 cpunode_mainbus_match(device_t parent, cfdata_t cf, void *aux) 98 { 99 100 return 1; 101 } 102 103 void 104 cpunode_mainbus_attach(device_t parent, device_t self, void *aux) 105 { 106 struct cpunode_softc * const sc = device_private(self); 107 int cpunum = 0; 108 109 sc->sc_dev = self; 110 sc->sc_fuse = octeon_xkphys_read_8(CIU_FUSE); 111 112 aprint_naive(": %u core%s\n", 113 popcount32((uint32_t)sc->sc_fuse), 114 sc->sc_fuse == 1 ? "" : "s"); 115 116 aprint_normal(": %u core%s", 117 popcount32((uint32_t)sc->sc_fuse), 118 sc->sc_fuse == 1 ? "" : "s"); 119 const uint64_t cvmctl = mips_cp0_cvmctl_read(); 120 aprint_normal(", %scrypto", (cvmctl & CP0_CVMCTL_NOCRYPTO) ? "no " : ""); 121 aprint_normal((cvmctl & CP0_CVMCTL_KASUMI) ? "+kasumi" : ""); 122 aprint_normal(", %s64bit-mul", (cvmctl & CP0_CVMCTL_NOMUL) ? "no " : ""); 123 if (cvmctl & CP0_CVMCTL_REPUN) 124 aprint_normal(", unaligned-access ok"); 125 #ifdef MULTIPROCESSOR 126 uint32_t booted[1]; 127 kcpuset_export_u32(cpus_booted, booted, sizeof(booted)); 128 aprint_normal(", booted %#" PRIx32, booted[0]); 129 #endif 130 aprint_normal("\n"); 131 132 for (uint64_t fuse = sc->sc_fuse; fuse != 0; fuse >>= 1, cpunum++) { 133 struct cpunode_attach_args cnaa = { 134 .cnaa_name = "cpu", 135 .cnaa_cpunum = cpunum, 136 }; 137 config_found(self, &cnaa, cpunode_mainbus_print); 138 } 139 #if NWDOG > 0 140 struct cpunode_attach_args cnaa = { 141 .cnaa_name = "wdog", 142 .cnaa_cpunum = CPUNODECF_CORE_DEFAULT, 143 }; 144 config_found(self, &cnaa, cpunode_mainbus_print); 145 #endif 146 } 147 148 int 149 cpu_cpunode_match(device_t parent, cfdata_t cf, void *aux) 150 { 151 struct cpunode_attach_args * const cnaa = aux; 152 const int cpunum = cf->cf_loc[CPUNODECF_CORE]; 153 154 return strcmp(cnaa->cnaa_name, cf->cf_name) == 0 155 && (cpunum == CPUNODECF_CORE_DEFAULT || cpunum == cnaa->cnaa_cpunum); 156 } 157 158 #if defined(MULTIPROCESSOR) 159 static bool 160 octeon_fixup_cpu_info_references(int32_t load_addr, uint32_t new_insns[2], 161 void *arg) 162 { 163 struct cpu_info * const ci = arg; 164 165 atomic_or_64(&curcpu()->ci_flags, CPUF_PRESENT); 166 167 KASSERT(MIPS_KSEG0_P(load_addr)); 168 #ifdef MULTIPROCESSOR 169 KASSERT(!CPU_IS_PRIMARY(curcpu())); 170 #endif 171 load_addr += (intptr_t)ci - (intptr_t)&cpu_info_store; 172 173 KASSERT((intptr_t)ci <= load_addr); 174 KASSERT(load_addr < (intptr_t)(ci + 1)); 175 176 KASSERT(INSN_LUI_P(new_insns[0])); 177 KASSERT(INSN_LOAD_P(new_insns[1]) || INSN_STORE_P(new_insns[1])); 178 179 /* 180 * Use the lui and load/store instruction as a prototype and 181 * make it refer to cpu1_info_store instead of cpu_info_store. 182 */ 183 new_insns[0] &= __BITS(31,16); 184 new_insns[1] &= __BITS(31,16); 185 new_insns[0] |= (uint16_t)((load_addr + 0x8000) >> 16); 186 new_insns[1] |= (uint16_t)load_addr; 187 #ifdef DEBUG_VERBOSE 188 printf("%s: %08x: insn#1 %08x: lui r%u, %d\n", 189 __func__, (int32_t)load_addr, new_insns[0], 190 (new_insns[0] >> 16) & 31, 191 (int16_t)new_insns[0]); 192 printf("%s: %08x: insn#2 %08x: %c%c r%u, %d(r%u)\n", 193 __func__, (int32_t)load_addr, new_insns[0], 194 INSN_LOAD_P(new_insns[1]) ? 'l' : 's', 195 INSN_LW_P(new_insns[1]) ? 'w' : 'd', 196 (new_insns[0] >> 16) & 31, 197 (int16_t)new_insns[1], 198 (new_insns[0] >> 21) & 31); 199 #endif 200 return true; 201 } 202 203 static void 204 octeon_cpu_init(struct cpu_info *ci) 205 { 206 bool ok __diagused; 207 208 // First thing is setup the execption vectors for this cpu. 209 mips64r2_vector_init(&mips_splsw); 210 211 // Next rewrite those exceptions to use this cpu's cpu_info. 212 ok = mips_fixup_exceptions(octeon_fixup_cpu_info_references, ci); 213 KASSERT(ok); 214 215 (void) splhigh(); // make sure interrupts are masked 216 217 KASSERT((mipsNN_cp0_ebase_read() & MIPS_EBASE_CPUNUM) == ci->ci_cpuid); 218 KASSERT(curcpu() == ci); 219 KASSERT(ci->ci_cpl == IPL_HIGH); 220 KASSERT((mips_cp0_status_read() & MIPS_INT_MASK) == 0); 221 } 222 223 static void 224 octeon_cpu_run(struct cpu_info *ci) 225 { 226 octeon_intr_init(ci); 227 228 mips3_initclocks(); 229 KASSERTMSG(ci->ci_cpl == IPL_NONE, "cpl %d", ci->ci_cpl); 230 KASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE); 231 232 aprint_normal("%s: ", device_xname(ci->ci_dev)); 233 cpu_identify(ci->ci_dev); 234 } 235 #endif /* MULTIPROCESSOR */ 236 237 static void 238 cpu_cpunode_attach_common(device_t self, struct cpu_info *ci) 239 { 240 struct cpu_softc * const cpu __diagused = ci->ci_softc; 241 242 ci->ci_dev = self; 243 self->dv_private = ci; 244 245 KASSERTMSG(cpu != NULL, "ci %p index %d", ci, cpu_index(ci)); 246 247 #if NWDOG > 0 || defined(DDB) 248 void **nmi_vector = (void *)MIPS_PHYS_TO_KSEG0(0x800 + 32*ci->ci_cpuid); 249 *nmi_vector = octeon_reset_vector; 250 251 struct vm_page * const pg = mips_pmap_alloc_poolpage(UVM_PGA_ZERO); 252 KASSERT(pg != NULL); 253 const vaddr_t kva = mips_pmap_map_poolpage(VM_PAGE_TO_PHYS(pg)); 254 KASSERT(kva != 0); 255 ci->ci_nmi_stack = (void *)(kva + PAGE_SIZE - sizeof(struct kernframe)); 256 #endif 257 258 #ifdef WDOG 259 cpu->cpu_wdog_sih = softint_establish(SOFTINT_CLOCK|SOFTINT_MPSAFE, 260 wdog_cpunode_poke, cpu); 261 KASSERT(cpu->cpu_wdog_sih != NULL); 262 #endif 263 264 aprint_normal(": %lu.%02luMHz (hz cycles = %lu, delay divisor = %lu)\n", 265 ci->ci_cpu_freq / 1000000, 266 (ci->ci_cpu_freq % 1000000) / 10000, 267 ci->ci_cycles_per_hz, ci->ci_divisor_delay); 268 269 if (CPU_IS_PRIMARY(ci)) { 270 aprint_normal("%s: ", device_xname(self)); 271 cpu_identify(self); 272 } 273 cpu_attach_common(self, ci); 274 #ifdef MULTIPROCESSOR 275 KASSERT(cpuid_infos[ci->ci_cpuid] == ci); 276 #endif 277 } 278 279 void 280 cpu_cpunode_attach(device_t parent, device_t self, void *aux) 281 { 282 struct cpunode_attach_args * const cnaa = aux; 283 const int cpunum = cnaa->cnaa_cpunum; 284 285 if (cpunum == 0) { 286 cpu_cpunode_attach_common(self, curcpu()); 287 #ifdef MULTIPROCESSOR 288 mips_locoresw.lsw_cpu_init = octeon_cpu_init; 289 mips_locoresw.lsw_cpu_run = octeon_cpu_run; 290 #endif 291 return; 292 } 293 #ifdef MULTIPROCESSOR 294 KASSERTMSG(cpunum == 1, "cpunum %d", cpunum); 295 if (!kcpuset_isset(cpus_booted, cpunum)) { 296 aprint_naive(" disabled\n"); 297 aprint_normal(" disabled (unresponsive)\n"); 298 return; 299 } 300 struct cpu_info * const ci = cpu_info_alloc(NULL, cpunum, 0, cpunum, 0); 301 302 ci->ci_softc = &octeon_cpu1_softc; 303 ci->ci_softc->cpu_ci = ci; 304 305 cpu_cpunode_attach_common(self, ci); 306 307 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 308 for (int i = 0; i < 100 && !kcpuset_isset(cpus_hatched, cpunum); i++) { 309 delay(10000); 310 } 311 if (!kcpuset_isset(cpus_hatched, cpunum)) { 312 #ifdef DDB 313 aprint_verbose_dev(self, "hatch failed ci=%p flags=%#"PRIx64"\n", ci, ci->ci_flags); 314 cpu_Debugger(); 315 #endif 316 panic("%s failed to hatch: ci=%p flags=%#"PRIx64, 317 cpu_name(ci), ci, ci->ci_flags); 318 } 319 #else 320 aprint_naive(": disabled\n"); 321 aprint_normal(": disabled (uniprocessor kernel)\n"); 322 #endif 323 } 324 325 #if NWDOG > 0 326 struct wdog_softc { 327 struct sysmon_wdog sc_smw; 328 device_t sc_dev; 329 u_int sc_wdog_period; 330 bool sc_wdog_armed; 331 }; 332 333 #ifndef OCTEON_WDOG_PERIOD_DEFAULT 334 #define OCTEON_WDOG_PERIOD_DEFAULT 4 335 #endif 336 337 static int wdog_cpunode_match(device_t, cfdata_t, void *); 338 static void wdog_cpunode_attach(device_t, device_t, void *); 339 340 CFATTACH_DECL_NEW(wdog_cpunode, sizeof(struct wdog_softc), 341 wdog_cpunode_match, wdog_cpunode_attach, NULL, NULL); 342 343 static int 344 wdog_cpunode_setmode(struct sysmon_wdog *smw) 345 { 346 struct wdog_softc * const sc = smw->smw_cookie; 347 348 if ((smw->smw_mode & WDOG_MODE_MASK) == WDOG_MODE_DISARMED) { 349 if (sc->sc_wdog_armed) { 350 CPU_INFO_ITERATOR cii; 351 struct cpu_info *ci; 352 for (CPU_INFO_FOREACH(cii, ci)) { 353 struct cpu_softc * const cpu = ci->ci_softc; 354 uint64_t wdog = mips64_ld_a64(cpu->cpu_wdog); 355 wdog &= ~CIU_WDOGX_MODE; 356 mips64_sd_a64(cpu->cpu_pp_poke, wdog); 357 aprint_verbose_dev(sc->sc_dev, 358 "%s: disable wdog=%#"PRIx64"\n", 359 cpu_name(ci), wdog); 360 mips64_sd_a64(cpu->cpu_wdog, wdog); 361 mips64_sd_a64(cpu->cpu_pp_poke, wdog); 362 } 363 sc->sc_wdog_armed = false; 364 } 365 } else if (!sc->sc_wdog_armed) { 366 kpreempt_disable(); 367 struct cpu_info *ci = curcpu(); 368 if (smw->smw_period == WDOG_PERIOD_DEFAULT) { 369 smw->smw_period = OCTEON_WDOG_PERIOD_DEFAULT; 370 } 371 uint64_t wdog_len = smw->smw_period * ci->ci_cpu_freq; 372 // 373 // This wdog is a 24-bit counter that decrements every 256 374 // cycles. This is then a 32-bit counter so as long wdog_len 375 // doesn't overflow a 32-bit value, we are fine. We write the 376 // 16-bits of the 32-bit period. 377 if ((wdog_len >> 32) != 0) { 378 kpreempt_enable(); 379 return EINVAL; 380 } 381 sc->sc_wdog_period = smw->smw_period; 382 CPU_INFO_ITERATOR cii; 383 for (CPU_INFO_FOREACH(cii, ci)) { 384 struct cpu_softc * const cpu = ci->ci_softc; 385 uint64_t wdog = mips64_ld_a64(cpu->cpu_wdog); 386 wdog &= ~(CIU_WDOGX_MODE|CIU_WDOGX_LEN); 387 wdog |= __SHIFTIN(3, CIU_WDOGX_MODE); 388 wdog |= __SHIFTIN(wdog_len >> 16, CIU_WDOGX_LEN); 389 aprint_verbose_dev(sc->sc_dev, 390 "%s: enable wdog=%#"PRIx64" (%#"PRIx64")\n", 391 cpu_name(ci), wdog, wdog_len); 392 mips64_sd_a64(cpu->cpu_wdog, wdog); 393 } 394 sc->sc_wdog_armed = true; 395 kpreempt_enable(); 396 } 397 return 0; 398 } 399 400 static void 401 wdog_cpunode_poke(void *arg) 402 { 403 struct cpu_softc *cpu = arg; 404 mips64_sd_a64(cpu->cpu_pp_poke, 0); 405 } 406 407 static int 408 wdog_cpunode_tickle(struct sysmon_wdog *smw) 409 { 410 wdog_cpunode_poke(curcpu()->ci_softc); 411 #ifdef MULTIPROCESSOR 412 // We need to send IPIs to the other CPUs to poke their wdog. 413 cpu_send_ipi(NULL, IPI_WDOG); 414 #endif 415 return 0; 416 } 417 418 int 419 wdog_cpunode_match(device_t parent, cfdata_t cf, void *aux) 420 { 421 struct cpunode_softc * const sc = device_private(parent); 422 struct cpunode_attach_args * const cnaa = aux; 423 const int cpunum = cf->cf_loc[CPUNODECF_CORE]; 424 425 return sc->sc_wdog_dev == NULL 426 && strcmp(cnaa->cnaa_name, cf->cf_name) == 0 427 && cpunum == CPUNODECF_CORE_DEFAULT; 428 } 429 430 void 431 wdog_cpunode_attach(device_t parent, device_t self, void *aux) 432 { 433 struct cpunode_softc * const psc = device_private(parent); 434 struct wdog_softc * const sc = device_private(self); 435 cfdata_t const cf = device_cfdata(self); 436 437 psc->sc_wdog_dev = self; 438 439 sc->sc_dev = self; 440 sc->sc_smw.smw_name = device_xname(self); 441 sc->sc_smw.smw_cookie = sc; 442 sc->sc_smw.smw_setmode = wdog_cpunode_setmode; 443 sc->sc_smw.smw_tickle = wdog_cpunode_tickle; 444 sc->sc_smw.smw_period = OCTEON_WDOG_PERIOD_DEFAULT; 445 sc->sc_wdog_period = sc->sc_smw.smw_period; 446 447 /* 448 * We need one softint per cpu. It's to tickle the softints on 449 * other CPUs. 450 */ 451 CPU_INFO_ITERATOR cii; 452 struct cpu_info *ci; 453 for (CPU_INFO_FOREACH(cii, ci)) { 454 } 455 456 aprint_normal(": default period is %u seconds%s\n", 457 sc->sc_wdog_period, sc->sc_wdog_period == 1 ? "" : "s"); 458 459 if (sysmon_wdog_register(&sc->sc_smw) != 0) { 460 aprint_error_dev(self, "unable to register with sysmon\n"); 461 return; 462 } 463 464 if (cf->cf_flags & 1) { 465 int error = sysmon_wdog_setmode(&sc->sc_smw, WDOG_MODE_KTICKLE, 466 sc->sc_wdog_period); 467 if (error) 468 aprint_error_dev(self, 469 "failed to start kernel tickler: %d\n", error); 470 } 471 } 472 #endif /* NWDOG > 0 */ 473