1 /* $NetBSD: kern_cpu.c,v 1.89 2019/12/21 11:35:25 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008, 2009, 2010, 2012, 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /*- 33 * Copyright (c)2007 YAMAMOTO Takashi, 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 */ 57 58 /* 59 * CPU related routines not shared with rump. 60 */ 61 62 #include <sys/cdefs.h> 63 __KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.89 2019/12/21 11:35:25 ad Exp $"); 64 65 #ifdef _KERNEL_OPT 66 #include "opt_cpu_ucode.h" 67 #endif 68 69 #include <sys/param.h> 70 #include <sys/systm.h> 71 #include <sys/idle.h> 72 #include <sys/sched.h> 73 #include <sys/intr.h> 74 #include <sys/conf.h> 75 #include <sys/cpu.h> 76 #include <sys/cpuio.h> 77 #include <sys/proc.h> 78 #include <sys/percpu.h> 79 #include <sys/kernel.h> 80 #include <sys/kauth.h> 81 #include <sys/xcall.h> 82 #include <sys/pool.h> 83 #include <sys/kmem.h> 84 #include <sys/select.h> 85 #include <sys/namei.h> 86 #include <sys/callout.h> 87 #include <sys/pcu.h> 88 89 #include <uvm/uvm_extern.h> 90 91 #include "ioconf.h" 92 93 /* 94 * If the port has stated that cpu_data is the first thing in cpu_info, 95 * verify that the claim is true. This will prevent them from getting out 96 * of sync. 97 */ 98 #ifdef __HAVE_CPU_DATA_FIRST 99 CTASSERT(offsetof(struct cpu_info, ci_data) == 0); 100 #else 101 CTASSERT(offsetof(struct cpu_info, ci_data) != 0); 102 #endif 103 104 int (*compat_cpuctl_ioctl)(struct lwp *, u_long, void *) = (void *)enosys; 105 106 static void cpu_xc_online(struct cpu_info *, void *); 107 static void cpu_xc_offline(struct cpu_info *, void *); 108 109 dev_type_ioctl(cpuctl_ioctl); 110 111 const struct cdevsw cpuctl_cdevsw = { 112 .d_open = nullopen, 113 .d_close = nullclose, 114 .d_read = nullread, 115 .d_write = nullwrite, 116 .d_ioctl = cpuctl_ioctl, 117 .d_stop = nullstop, 118 .d_tty = notty, 119 .d_poll = nopoll, 120 .d_mmap = nommap, 121 .d_kqfilter = nokqfilter, 122 .d_discard = nodiscard, 123 .d_flag = D_OTHER | D_MPSAFE 124 }; 125 126 int 127 mi_cpu_attach(struct cpu_info *ci) 128 { 129 int error; 130 131 KASSERT(maxcpus > 0); 132 133 ci->ci_index = ncpu; 134 kcpuset_set(kcpuset_attached, cpu_index(ci)); 135 136 /* 137 * Create a convenience cpuset of just ourselves. 138 */ 139 kcpuset_create(&ci->ci_data.cpu_kcpuset, true); 140 kcpuset_set(ci->ci_data.cpu_kcpuset, cpu_index(ci)); 141 142 TAILQ_INIT(&ci->ci_data.cpu_ld_locks); 143 __cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock); 144 145 /* This is useful for eg, per-cpu evcnt */ 146 snprintf(ci->ci_data.cpu_name, sizeof(ci->ci_data.cpu_name), "cpu%d", 147 cpu_index(ci)); 148 149 if (__predict_false(cpu_infos == NULL)) { 150 size_t ci_bufsize = (maxcpus + 1) * sizeof(struct cpu_info *); 151 cpu_infos = kmem_zalloc(ci_bufsize, KM_SLEEP); 152 } 153 cpu_infos[cpu_index(ci)] = ci; 154 155 sched_cpuattach(ci); 156 157 error = create_idle_lwp(ci); 158 if (error != 0) { 159 /* XXX revert sched_cpuattach */ 160 return error; 161 } 162 163 if (ci == curcpu()) 164 ci->ci_onproc = curlwp; 165 else 166 ci->ci_onproc = ci->ci_data.cpu_idlelwp; 167 168 percpu_init_cpu(ci); 169 softint_init(ci); 170 callout_init_cpu(ci); 171 xc_init_cpu(ci); 172 pool_cache_cpu_init(ci); 173 selsysinit(ci); 174 cache_cpu_init(ci); 175 TAILQ_INIT(&ci->ci_data.cpu_biodone); 176 ncpu++; 177 ncpuonline++; 178 179 return 0; 180 } 181 182 void 183 cpuctlattach(int dummy __unused) 184 { 185 186 KASSERT(cpu_infos != NULL); 187 } 188 189 int 190 cpuctl_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l) 191 { 192 CPU_INFO_ITERATOR cii; 193 cpustate_t *cs; 194 struct cpu_info *ci; 195 int error, i; 196 u_int id; 197 198 error = 0; 199 200 mutex_enter(&cpu_lock); 201 switch (cmd) { 202 case IOC_CPU_SETSTATE: 203 cs = data; 204 error = kauth_authorize_system(l->l_cred, 205 KAUTH_SYSTEM_CPU, KAUTH_REQ_SYSTEM_CPU_SETSTATE, cs, NULL, 206 NULL); 207 if (error != 0) 208 break; 209 if (cs->cs_id >= maxcpus || 210 (ci = cpu_lookup(cs->cs_id)) == NULL) { 211 error = ESRCH; 212 break; 213 } 214 cpu_setintr(ci, cs->cs_intr); 215 error = cpu_setstate(ci, cs->cs_online); 216 break; 217 218 case IOC_CPU_GETSTATE: 219 cs = data; 220 id = cs->cs_id; 221 memset(cs, 0, sizeof(*cs)); 222 cs->cs_id = id; 223 if (cs->cs_id >= maxcpus || 224 (ci = cpu_lookup(id)) == NULL) { 225 error = ESRCH; 226 break; 227 } 228 if ((ci->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 229 cs->cs_online = false; 230 else 231 cs->cs_online = true; 232 if ((ci->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 233 cs->cs_intr = false; 234 else 235 cs->cs_intr = true; 236 cs->cs_lastmod = (int32_t)ci->ci_schedstate.spc_lastmod; 237 cs->cs_lastmodhi = (int32_t) 238 (ci->ci_schedstate.spc_lastmod >> 32); 239 cs->cs_intrcnt = cpu_intr_count(ci) + 1; 240 cs->cs_hwid = ci->ci_cpuid; 241 break; 242 243 case IOC_CPU_MAPID: 244 i = 0; 245 for (CPU_INFO_FOREACH(cii, ci)) { 246 if (i++ == *(int *)data) 247 break; 248 } 249 if (ci == NULL) 250 error = ESRCH; 251 else 252 *(int *)data = cpu_index(ci); 253 break; 254 255 case IOC_CPU_GETCOUNT: 256 *(int *)data = ncpu; 257 break; 258 259 #ifdef CPU_UCODE 260 case IOC_CPU_UCODE_GET_VERSION: 261 error = cpu_ucode_get_version((struct cpu_ucode_version *)data); 262 break; 263 264 case IOC_CPU_UCODE_APPLY: 265 error = kauth_authorize_machdep(l->l_cred, 266 KAUTH_MACHDEP_CPU_UCODE_APPLY, 267 NULL, NULL, NULL, NULL); 268 if (error != 0) 269 break; 270 error = cpu_ucode_apply((const struct cpu_ucode *)data); 271 break; 272 #endif 273 274 default: 275 error = (*compat_cpuctl_ioctl)(l, cmd, data); 276 break; 277 } 278 mutex_exit(&cpu_lock); 279 280 return error; 281 } 282 283 struct cpu_info * 284 cpu_lookup(u_int idx) 285 { 286 struct cpu_info *ci; 287 288 /* 289 * cpu_infos is a NULL terminated array of MAXCPUS + 1 entries, 290 * so an index of MAXCPUS here is ok. See mi_cpu_attach. 291 */ 292 KASSERT(idx <= maxcpus); 293 294 if (__predict_false(cpu_infos == NULL)) { 295 KASSERT(idx == 0); 296 return curcpu(); 297 } 298 299 ci = cpu_infos[idx]; 300 KASSERT(ci == NULL || cpu_index(ci) == idx); 301 KASSERTMSG(idx < maxcpus || ci == NULL, "idx %d ci %p", idx, ci); 302 303 return ci; 304 } 305 306 static void 307 cpu_xc_offline(struct cpu_info *ci, void *unused) 308 { 309 struct schedstate_percpu *spc, *mspc = NULL; 310 struct cpu_info *target_ci; 311 struct lwp *l; 312 CPU_INFO_ITERATOR cii; 313 int s; 314 315 /* 316 * Thread that made the cross call (separate context) holds 317 * cpu_lock on our behalf. 318 */ 319 spc = &ci->ci_schedstate; 320 s = splsched(); 321 spc->spc_flags |= SPCF_OFFLINE; 322 splx(s); 323 324 /* Take the first available CPU for the migration. */ 325 for (CPU_INFO_FOREACH(cii, target_ci)) { 326 mspc = &target_ci->ci_schedstate; 327 if ((mspc->spc_flags & SPCF_OFFLINE) == 0) 328 break; 329 } 330 KASSERT(target_ci != NULL); 331 332 /* 333 * Migrate all non-bound threads to the other CPU. Note that this 334 * runs from the xcall thread, thus handling of LSONPROC is not needed. 335 */ 336 mutex_enter(proc_lock); 337 LIST_FOREACH(l, &alllwp, l_list) { 338 struct cpu_info *mci; 339 340 lwp_lock(l); 341 if (l->l_cpu != ci || (l->l_pflag & (LP_BOUND | LP_INTR))) { 342 lwp_unlock(l); 343 continue; 344 } 345 /* Regular case - no affinity. */ 346 if (l->l_affinity == NULL) { 347 lwp_migrate(l, target_ci); 348 continue; 349 } 350 /* Affinity is set, find an online CPU in the set. */ 351 for (CPU_INFO_FOREACH(cii, mci)) { 352 mspc = &mci->ci_schedstate; 353 if ((mspc->spc_flags & SPCF_OFFLINE) == 0 && 354 kcpuset_isset(l->l_affinity, cpu_index(mci))) 355 break; 356 } 357 if (mci == NULL) { 358 lwp_unlock(l); 359 mutex_exit(proc_lock); 360 goto fail; 361 } 362 lwp_migrate(l, mci); 363 } 364 mutex_exit(proc_lock); 365 366 #if PCU_UNIT_COUNT > 0 367 pcu_save_all_on_cpu(); 368 #endif 369 370 #ifdef __HAVE_MD_CPU_OFFLINE 371 cpu_offline_md(); 372 #endif 373 return; 374 fail: 375 /* Just unset the SPCF_OFFLINE flag, caller will check */ 376 s = splsched(); 377 spc->spc_flags &= ~SPCF_OFFLINE; 378 splx(s); 379 } 380 381 static void 382 cpu_xc_online(struct cpu_info *ci, void *unused) 383 { 384 struct schedstate_percpu *spc; 385 int s; 386 387 spc = &ci->ci_schedstate; 388 s = splsched(); 389 spc->spc_flags &= ~SPCF_OFFLINE; 390 splx(s); 391 } 392 393 int 394 cpu_setstate(struct cpu_info *ci, bool online) 395 { 396 struct schedstate_percpu *spc; 397 CPU_INFO_ITERATOR cii; 398 struct cpu_info *ci2; 399 uint64_t where; 400 xcfunc_t func; 401 int nonline; 402 403 spc = &ci->ci_schedstate; 404 405 KASSERT(mutex_owned(&cpu_lock)); 406 407 if (online) { 408 if ((spc->spc_flags & SPCF_OFFLINE) == 0) 409 return 0; 410 func = (xcfunc_t)cpu_xc_online; 411 } else { 412 if ((spc->spc_flags & SPCF_OFFLINE) != 0) 413 return 0; 414 nonline = 0; 415 /* 416 * Ensure that at least one CPU within the processor set 417 * stays online. Revisit this later. 418 */ 419 for (CPU_INFO_FOREACH(cii, ci2)) { 420 if ((ci2->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 421 continue; 422 if (ci2->ci_schedstate.spc_psid != spc->spc_psid) 423 continue; 424 nonline++; 425 } 426 if (nonline == 1) 427 return EBUSY; 428 func = (xcfunc_t)cpu_xc_offline; 429 } 430 431 where = xc_unicast(0, func, ci, NULL, ci); 432 xc_wait(where); 433 if (online) { 434 KASSERT((spc->spc_flags & SPCF_OFFLINE) == 0); 435 ncpuonline++; 436 } else { 437 if ((spc->spc_flags & SPCF_OFFLINE) == 0) { 438 /* If was not set offline, then it is busy */ 439 return EBUSY; 440 } 441 ncpuonline--; 442 } 443 444 spc->spc_lastmod = time_second; 445 return 0; 446 } 447 448 #if defined(__HAVE_INTR_CONTROL) 449 static void 450 cpu_xc_intr(struct cpu_info *ci, void *unused) 451 { 452 struct schedstate_percpu *spc; 453 int s; 454 455 spc = &ci->ci_schedstate; 456 s = splsched(); 457 spc->spc_flags &= ~SPCF_NOINTR; 458 splx(s); 459 } 460 461 static void 462 cpu_xc_nointr(struct cpu_info *ci, void *unused) 463 { 464 struct schedstate_percpu *spc; 465 int s; 466 467 spc = &ci->ci_schedstate; 468 s = splsched(); 469 spc->spc_flags |= SPCF_NOINTR; 470 splx(s); 471 } 472 473 int 474 cpu_setintr(struct cpu_info *ci, bool intr) 475 { 476 struct schedstate_percpu *spc; 477 CPU_INFO_ITERATOR cii; 478 struct cpu_info *ci2; 479 uint64_t where; 480 xcfunc_t func; 481 int nintr; 482 483 spc = &ci->ci_schedstate; 484 485 KASSERT(mutex_owned(&cpu_lock)); 486 487 if (intr) { 488 if ((spc->spc_flags & SPCF_NOINTR) == 0) 489 return 0; 490 func = (xcfunc_t)cpu_xc_intr; 491 } else { 492 if ((spc->spc_flags & SPCF_NOINTR) != 0) 493 return 0; 494 /* 495 * Ensure that at least one CPU within the system 496 * is handing device interrupts. 497 */ 498 nintr = 0; 499 for (CPU_INFO_FOREACH(cii, ci2)) { 500 if ((ci2->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 501 continue; 502 if (ci2 == ci) 503 continue; 504 nintr++; 505 } 506 if (nintr == 0) 507 return EBUSY; 508 func = (xcfunc_t)cpu_xc_nointr; 509 } 510 511 where = xc_unicast(0, func, ci, NULL, ci); 512 xc_wait(where); 513 if (intr) { 514 KASSERT((spc->spc_flags & SPCF_NOINTR) == 0); 515 } else if ((spc->spc_flags & SPCF_NOINTR) == 0) { 516 /* If was not set offline, then it is busy */ 517 return EBUSY; 518 } 519 520 /* Direct interrupts away from the CPU and record the change. */ 521 cpu_intr_redistribute(); 522 spc->spc_lastmod = time_second; 523 return 0; 524 } 525 #else /* __HAVE_INTR_CONTROL */ 526 int 527 cpu_setintr(struct cpu_info *ci, bool intr) 528 { 529 530 return EOPNOTSUPP; 531 } 532 533 u_int 534 cpu_intr_count(struct cpu_info *ci) 535 { 536 537 return 0; /* 0 == "don't know" */ 538 } 539 #endif /* __HAVE_INTR_CONTROL */ 540 541 #ifdef CPU_UCODE 542 int 543 cpu_ucode_load(struct cpu_ucode_softc *sc, const char *fwname) 544 { 545 firmware_handle_t fwh; 546 int error; 547 548 if (sc->sc_blob != NULL) { 549 firmware_free(sc->sc_blob, sc->sc_blobsize); 550 sc->sc_blob = NULL; 551 sc->sc_blobsize = 0; 552 } 553 554 error = cpu_ucode_md_open(&fwh, sc->loader_version, fwname); 555 if (error != 0) { 556 #ifdef DEBUG 557 printf("ucode: firmware_open(%s) failed: %i\n", fwname, error); 558 #endif 559 goto err0; 560 } 561 562 sc->sc_blobsize = firmware_get_size(fwh); 563 if (sc->sc_blobsize == 0) { 564 error = EFTYPE; 565 firmware_close(fwh); 566 goto err0; 567 } 568 sc->sc_blob = firmware_malloc(sc->sc_blobsize); 569 if (sc->sc_blob == NULL) { 570 error = ENOMEM; 571 firmware_close(fwh); 572 goto err0; 573 } 574 575 error = firmware_read(fwh, 0, sc->sc_blob, sc->sc_blobsize); 576 firmware_close(fwh); 577 if (error != 0) 578 goto err1; 579 580 return 0; 581 582 err1: 583 firmware_free(sc->sc_blob, sc->sc_blobsize); 584 sc->sc_blob = NULL; 585 sc->sc_blobsize = 0; 586 err0: 587 return error; 588 } 589 #endif 590