1 /* $NetBSD: kern_cpu.c,v 1.54 2012/01/17 10:47:27 cegger Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008, 2009, 2010, 2012 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /*- 33 * Copyright (c)2007 YAMAMOTO Takashi, 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 */ 57 58 #include <sys/cdefs.h> 59 __KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.54 2012/01/17 10:47:27 cegger Exp $"); 60 61 #include "opt_cpu_ucode.h" 62 63 #include <sys/param.h> 64 #include <sys/systm.h> 65 #include <sys/idle.h> 66 #include <sys/sched.h> 67 #include <sys/intr.h> 68 #include <sys/conf.h> 69 #include <sys/cpu.h> 70 #include <sys/cpuio.h> 71 #include <sys/proc.h> 72 #include <sys/percpu.h> 73 #include <sys/kernel.h> 74 #include <sys/kauth.h> 75 #include <sys/xcall.h> 76 #include <sys/pool.h> 77 #include <sys/kmem.h> 78 #include <sys/select.h> 79 #include <sys/namei.h> 80 #include <sys/callout.h> 81 82 #include <uvm/uvm_extern.h> 83 84 /* 85 * If the port has stated that cpu_data is the first thing in cpu_info, 86 * verify that the claim is true. This will prevent them from getting out 87 * of sync. 88 */ 89 #ifdef __HAVE_CPU_DATA_FIRST 90 CTASSERT(offsetof(struct cpu_info, ci_data) == 0); 91 #else 92 CTASSERT(offsetof(struct cpu_info, ci_data) != 0); 93 #endif 94 95 void cpuctlattach(int); 96 97 static void cpu_xc_online(struct cpu_info *); 98 static void cpu_xc_offline(struct cpu_info *); 99 100 dev_type_ioctl(cpuctl_ioctl); 101 102 const struct cdevsw cpuctl_cdevsw = { 103 nullopen, nullclose, nullread, nullwrite, cpuctl_ioctl, 104 nullstop, notty, nopoll, nommap, nokqfilter, 105 D_OTHER | D_MPSAFE 106 }; 107 108 kmutex_t cpu_lock __cacheline_aligned; 109 int ncpu __read_mostly; 110 int ncpuonline __read_mostly; 111 bool mp_online __read_mostly; 112 113 kcpuset_t * kcpuset_attached __read_mostly; 114 115 struct cpuqueue cpu_queue __cacheline_aligned 116 = CIRCLEQ_HEAD_INITIALIZER(cpu_queue); 117 118 static struct cpu_info **cpu_infos __read_mostly; 119 120 int 121 mi_cpu_attach(struct cpu_info *ci) 122 { 123 int error; 124 125 KASSERT(maxcpus > 0); 126 127 ci->ci_index = ncpu; 128 CIRCLEQ_INSERT_TAIL(&cpu_queue, ci, ci_data.cpu_qchain); 129 TAILQ_INIT(&ci->ci_data.cpu_ld_locks); 130 __cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock); 131 132 /* This is useful for eg, per-cpu evcnt */ 133 snprintf(ci->ci_data.cpu_name, sizeof(ci->ci_data.cpu_name), "cpu%d", 134 cpu_index(ci)); 135 136 if (__predict_false(cpu_infos == NULL)) { 137 cpu_infos = 138 kmem_zalloc(sizeof(cpu_infos[0]) * maxcpus, KM_SLEEP); 139 kcpuset_create(&kcpuset_attached, true); 140 } 141 cpu_infos[cpu_index(ci)] = ci; 142 kcpuset_set(kcpuset_attached, ci->ci_index); 143 144 sched_cpuattach(ci); 145 146 error = create_idle_lwp(ci); 147 if (error != 0) { 148 /* XXX revert sched_cpuattach */ 149 return error; 150 } 151 152 if (ci == curcpu()) 153 ci->ci_data.cpu_onproc = curlwp; 154 else 155 ci->ci_data.cpu_onproc = ci->ci_data.cpu_idlelwp; 156 157 percpu_init_cpu(ci); 158 softint_init(ci); 159 callout_init_cpu(ci); 160 xc_init_cpu(ci); 161 pool_cache_cpu_init(ci); 162 selsysinit(ci); 163 cache_cpu_init(ci); 164 TAILQ_INIT(&ci->ci_data.cpu_biodone); 165 ncpu++; 166 ncpuonline++; 167 168 return 0; 169 } 170 171 void 172 cpuctlattach(int dummy) 173 { 174 175 KASSERT(cpu_infos != NULL); 176 } 177 178 int 179 cpuctl_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l) 180 { 181 CPU_INFO_ITERATOR cii; 182 cpustate_t *cs; 183 struct cpu_info *ci; 184 int error, i; 185 u_int id; 186 187 error = 0; 188 189 mutex_enter(&cpu_lock); 190 switch (cmd) { 191 case IOC_CPU_SETSTATE: 192 if (error == 0) 193 cs = data; 194 error = kauth_authorize_system(l->l_cred, 195 KAUTH_SYSTEM_CPU, KAUTH_REQ_SYSTEM_CPU_SETSTATE, cs, NULL, 196 NULL); 197 if (error != 0) 198 break; 199 if (cs->cs_id >= maxcpus || 200 (ci = cpu_lookup(cs->cs_id)) == NULL) { 201 error = ESRCH; 202 break; 203 } 204 error = cpu_setintr(ci, cs->cs_intr); 205 error = cpu_setstate(ci, cs->cs_online); 206 break; 207 208 case IOC_CPU_GETSTATE: 209 if (error == 0) 210 cs = data; 211 id = cs->cs_id; 212 memset(cs, 0, sizeof(*cs)); 213 cs->cs_id = id; 214 if (cs->cs_id >= maxcpus || 215 (ci = cpu_lookup(id)) == NULL) { 216 error = ESRCH; 217 break; 218 } 219 if ((ci->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 220 cs->cs_online = false; 221 else 222 cs->cs_online = true; 223 if ((ci->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 224 cs->cs_intr = false; 225 else 226 cs->cs_intr = true; 227 cs->cs_lastmod = (int32_t)ci->ci_schedstate.spc_lastmod; 228 cs->cs_lastmodhi = (int32_t) 229 (ci->ci_schedstate.spc_lastmod >> 32); 230 cs->cs_intrcnt = cpu_intr_count(ci) + 1; 231 cs->cs_hwid = ci->ci_cpuid; 232 break; 233 234 case IOC_CPU_MAPID: 235 i = 0; 236 for (CPU_INFO_FOREACH(cii, ci)) { 237 if (i++ == *(int *)data) 238 break; 239 } 240 if (ci == NULL) 241 error = ESRCH; 242 else 243 *(int *)data = cpu_index(ci); 244 break; 245 246 case IOC_CPU_GETCOUNT: 247 *(int *)data = ncpu; 248 break; 249 250 #ifdef CPU_UCODE 251 case IOC_CPU_UCODE_GET_VERSION: 252 error = cpu_ucode_get_version(data); 253 break; 254 255 case IOC_CPU_UCODE_APPLY: 256 error = kauth_authorize_machdep(l->l_cred, 257 KAUTH_MACHDEP_CPU_UCODE_APPLY, 258 NULL, NULL, NULL, NULL); 259 if (error != 0) 260 break; 261 error = cpu_ucode_apply(data); 262 break; 263 #endif 264 265 default: 266 error = ENOTTY; 267 break; 268 } 269 mutex_exit(&cpu_lock); 270 271 return error; 272 } 273 274 struct cpu_info * 275 cpu_lookup(u_int idx) 276 { 277 struct cpu_info *ci; 278 279 KASSERT(idx < maxcpus); 280 281 if (__predict_false(cpu_infos == NULL)) { 282 KASSERT(idx == 0); 283 return curcpu(); 284 } 285 286 ci = cpu_infos[idx]; 287 KASSERT(ci == NULL || cpu_index(ci) == idx); 288 289 return ci; 290 } 291 292 static void 293 cpu_xc_offline(struct cpu_info *ci) 294 { 295 struct schedstate_percpu *spc, *mspc = NULL; 296 struct cpu_info *target_ci; 297 struct lwp *l; 298 CPU_INFO_ITERATOR cii; 299 int s; 300 301 /* 302 * Thread that made the cross call (separate context) holds 303 * cpu_lock on our behalf. 304 */ 305 spc = &ci->ci_schedstate; 306 s = splsched(); 307 spc->spc_flags |= SPCF_OFFLINE; 308 splx(s); 309 310 /* Take the first available CPU for the migration. */ 311 for (CPU_INFO_FOREACH(cii, target_ci)) { 312 mspc = &target_ci->ci_schedstate; 313 if ((mspc->spc_flags & SPCF_OFFLINE) == 0) 314 break; 315 } 316 KASSERT(target_ci != NULL); 317 318 /* 319 * Migrate all non-bound threads to the other CPU. Note that this 320 * runs from the xcall thread, thus handling of LSONPROC is not needed. 321 */ 322 mutex_enter(proc_lock); 323 LIST_FOREACH(l, &alllwp, l_list) { 324 struct cpu_info *mci; 325 326 lwp_lock(l); 327 if (l->l_cpu != ci || (l->l_pflag & (LP_BOUND | LP_INTR))) { 328 lwp_unlock(l); 329 continue; 330 } 331 /* Regular case - no affinity. */ 332 if (l->l_affinity == NULL) { 333 lwp_migrate(l, target_ci); 334 continue; 335 } 336 /* Affinity is set, find an online CPU in the set. */ 337 for (CPU_INFO_FOREACH(cii, mci)) { 338 mspc = &mci->ci_schedstate; 339 if ((mspc->spc_flags & SPCF_OFFLINE) == 0 && 340 kcpuset_isset(l->l_affinity, cpu_index(mci))) 341 break; 342 } 343 if (mci == NULL) { 344 lwp_unlock(l); 345 mutex_exit(proc_lock); 346 goto fail; 347 } 348 lwp_migrate(l, mci); 349 } 350 mutex_exit(proc_lock); 351 352 #ifdef __HAVE_MD_CPU_OFFLINE 353 cpu_offline_md(); 354 #endif 355 return; 356 fail: 357 /* Just unset the SPCF_OFFLINE flag, caller will check */ 358 s = splsched(); 359 spc->spc_flags &= ~SPCF_OFFLINE; 360 splx(s); 361 } 362 363 static void 364 cpu_xc_online(struct cpu_info *ci) 365 { 366 struct schedstate_percpu *spc; 367 int s; 368 369 spc = &ci->ci_schedstate; 370 s = splsched(); 371 spc->spc_flags &= ~SPCF_OFFLINE; 372 splx(s); 373 } 374 375 int 376 cpu_setstate(struct cpu_info *ci, bool online) 377 { 378 struct schedstate_percpu *spc; 379 CPU_INFO_ITERATOR cii; 380 struct cpu_info *ci2; 381 uint64_t where; 382 xcfunc_t func; 383 int nonline; 384 385 spc = &ci->ci_schedstate; 386 387 KASSERT(mutex_owned(&cpu_lock)); 388 389 if (online) { 390 if ((spc->spc_flags & SPCF_OFFLINE) == 0) 391 return 0; 392 func = (xcfunc_t)cpu_xc_online; 393 ncpuonline++; 394 } else { 395 if ((spc->spc_flags & SPCF_OFFLINE) != 0) 396 return 0; 397 nonline = 0; 398 /* 399 * Ensure that at least one CPU within the processor set 400 * stays online. Revisit this later. 401 */ 402 for (CPU_INFO_FOREACH(cii, ci2)) { 403 if ((ci2->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 404 continue; 405 if (ci2->ci_schedstate.spc_psid != spc->spc_psid) 406 continue; 407 nonline++; 408 } 409 if (nonline == 1) 410 return EBUSY; 411 func = (xcfunc_t)cpu_xc_offline; 412 ncpuonline--; 413 } 414 415 where = xc_unicast(0, func, ci, NULL, ci); 416 xc_wait(where); 417 if (online) { 418 KASSERT((spc->spc_flags & SPCF_OFFLINE) == 0); 419 } else if ((spc->spc_flags & SPCF_OFFLINE) == 0) { 420 /* If was not set offline, then it is busy */ 421 return EBUSY; 422 } 423 424 spc->spc_lastmod = time_second; 425 return 0; 426 } 427 428 #ifdef __HAVE_INTR_CONTROL 429 static void 430 cpu_xc_intr(struct cpu_info *ci) 431 { 432 struct schedstate_percpu *spc; 433 int s; 434 435 spc = &ci->ci_schedstate; 436 s = splsched(); 437 spc->spc_flags &= ~SPCF_NOINTR; 438 splx(s); 439 } 440 441 static void 442 cpu_xc_nointr(struct cpu_info *ci) 443 { 444 struct schedstate_percpu *spc; 445 int s; 446 447 spc = &ci->ci_schedstate; 448 s = splsched(); 449 spc->spc_flags |= SPCF_NOINTR; 450 splx(s); 451 } 452 453 int 454 cpu_setintr(struct cpu_info *ci, bool intr) 455 { 456 struct schedstate_percpu *spc; 457 CPU_INFO_ITERATOR cii; 458 struct cpu_info *ci2; 459 uint64_t where; 460 xcfunc_t func; 461 int nintr; 462 463 spc = &ci->ci_schedstate; 464 465 KASSERT(mutex_owned(&cpu_lock)); 466 467 if (intr) { 468 if ((spc->spc_flags & SPCF_NOINTR) == 0) 469 return 0; 470 func = (xcfunc_t)cpu_xc_intr; 471 } else { 472 if ((spc->spc_flags & SPCF_NOINTR) != 0) 473 return 0; 474 /* 475 * Ensure that at least one CPU within the system 476 * is handing device interrupts. 477 */ 478 nintr = 0; 479 for (CPU_INFO_FOREACH(cii, ci2)) { 480 if ((ci2->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 481 continue; 482 if (ci2 == ci) 483 continue; 484 nintr++; 485 } 486 if (nintr == 0) 487 return EBUSY; 488 func = (xcfunc_t)cpu_xc_nointr; 489 } 490 491 where = xc_unicast(0, func, ci, NULL, ci); 492 xc_wait(where); 493 if (intr) { 494 KASSERT((spc->spc_flags & SPCF_NOINTR) == 0); 495 } else if ((spc->spc_flags & SPCF_NOINTR) == 0) { 496 /* If was not set offline, then it is busy */ 497 return EBUSY; 498 } 499 500 /* Direct interrupts away from the CPU and record the change. */ 501 cpu_intr_redistribute(); 502 spc->spc_lastmod = time_second; 503 return 0; 504 } 505 #else /* __HAVE_INTR_CONTROL */ 506 int 507 cpu_setintr(struct cpu_info *ci, bool intr) 508 { 509 510 return EOPNOTSUPP; 511 } 512 513 u_int 514 cpu_intr_count(struct cpu_info *ci) 515 { 516 517 return 0; /* 0 == "don't know" */ 518 } 519 #endif /* __HAVE_INTR_CONTROL */ 520 521 bool 522 cpu_softintr_p(void) 523 { 524 525 return (curlwp->l_pflag & LP_INTR) != 0; 526 } 527 528 #ifdef CPU_UCODE 529 int 530 cpu_ucode_load(struct cpu_ucode_softc *sc, const char *fwname) 531 { 532 firmware_handle_t fwh; 533 int error; 534 535 if (sc->sc_blob != NULL) { 536 firmware_free(sc->sc_blob, 0); 537 sc->sc_blob = NULL; 538 sc->sc_blobsize = 0; 539 } 540 541 error = cpu_ucode_md_open(&fwh, fwname); 542 if (error != 0) { 543 aprint_error("ucode: firmware_open failed: %i\n", error); 544 goto err0; 545 } 546 547 sc->sc_blobsize = firmware_get_size(fwh); 548 sc->sc_blob = firmware_malloc(sc->sc_blobsize); 549 if (sc->sc_blob == NULL) { 550 error = ENOMEM; 551 firmware_close(fwh); 552 goto err0; 553 } 554 555 error = firmware_read(fwh, 0, sc->sc_blob, sc->sc_blobsize); 556 firmware_close(fwh); 557 if (error != 0) 558 goto err1; 559 560 return 0; 561 562 err1: 563 firmware_free(sc->sc_blob, 0); 564 sc->sc_blob = NULL; 565 sc->sc_blobsize = 0; 566 err0: 567 return error; 568 } 569 #endif 570