1 /* $NetBSD: subr_cpu.c,v 1.15 2020/06/11 22:21:05 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008, 2009, 2010, 2012, 2019, 2020 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Andrew Doran. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /*- 34 * Copyright (c)2007 YAMAMOTO Takashi, 35 * All rights reserved. 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 1. Redistributions of source code must retain the above copyright 41 * notice, this list of conditions and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 46 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 47 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 48 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 49 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 50 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 51 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 52 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 53 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 54 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 55 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 56 * SUCH DAMAGE. 57 */ 58 59 /* 60 * CPU related routines shared with rump. 61 */ 62 63 #include <sys/cdefs.h> 64 __KERNEL_RCSID(0, "$NetBSD: subr_cpu.c,v 1.15 2020/06/11 22:21:05 ad Exp $"); 65 66 #include <sys/param.h> 67 #include <sys/atomic.h> 68 #include <sys/systm.h> 69 #include <sys/sched.h> 70 #include <sys/conf.h> 71 #include <sys/cpu.h> 72 #include <sys/proc.h> 73 #include <sys/kernel.h> 74 #include <sys/kmem.h> 75 76 static void cpu_topology_fake1(struct cpu_info *); 77 78 kmutex_t cpu_lock __cacheline_aligned; 79 int ncpu __read_mostly; 80 int ncpuonline __read_mostly; 81 bool mp_online __read_mostly; 82 static bool cpu_topology_present __read_mostly; 83 static bool cpu_topology_haveslow __read_mostly; 84 int64_t cpu_counts[CPU_COUNT_MAX]; 85 86 /* An array of CPUs. There are ncpu entries. */ 87 struct cpu_info **cpu_infos __read_mostly; 88 89 /* Note: set on mi_cpu_attach() and idle_loop(). */ 90 kcpuset_t * kcpuset_attached __read_mostly = NULL; 91 kcpuset_t * kcpuset_running __read_mostly = NULL; 92 93 static char cpu_model[128]; 94 95 /* 96 * mi_cpu_init: early initialisation of MI CPU related structures. 97 * 98 * Note: may not block and memory allocator is not yet available. 99 */ 100 void 101 mi_cpu_init(void) 102 { 103 struct cpu_info *ci; 104 105 mutex_init(&cpu_lock, MUTEX_DEFAULT, IPL_NONE); 106 107 kcpuset_create(&kcpuset_attached, true); 108 kcpuset_create(&kcpuset_running, true); 109 kcpuset_set(kcpuset_running, 0); 110 111 ci = curcpu(); 112 cpu_topology_fake1(ci); 113 } 114 115 int 116 cpu_setmodel(const char *fmt, ...) 117 { 118 int len; 119 va_list ap; 120 121 va_start(ap, fmt); 122 len = vsnprintf(cpu_model, sizeof(cpu_model), fmt, ap); 123 va_end(ap); 124 return len; 125 } 126 127 const char * 128 cpu_getmodel(void) 129 { 130 return cpu_model; 131 } 132 133 bool 134 cpu_softintr_p(void) 135 { 136 137 return (curlwp->l_pflag & LP_INTR) != 0; 138 } 139 140 /* 141 * Collect CPU topology information as each CPU is attached. This can be 142 * called early during boot, so we need to be careful what we do. 143 */ 144 void 145 cpu_topology_set(struct cpu_info *ci, u_int package_id, u_int core_id, 146 u_int smt_id, u_int numa_id) 147 { 148 enum cpu_rel rel; 149 150 cpu_topology_present = true; 151 ci->ci_package_id = package_id; 152 ci->ci_core_id = core_id; 153 ci->ci_smt_id = smt_id; 154 ci->ci_numa_id = numa_id; 155 for (rel = 0; rel < __arraycount(ci->ci_sibling); rel++) { 156 ci->ci_sibling[rel] = ci; 157 ci->ci_nsibling[rel] = 1; 158 } 159 } 160 161 /* 162 * Collect CPU relative speed 163 */ 164 void 165 cpu_topology_setspeed(struct cpu_info *ci, bool slow) 166 { 167 168 cpu_topology_haveslow |= slow; 169 ci->ci_is_slow = slow; 170 } 171 172 /* 173 * Link a CPU into the given circular list. 174 */ 175 static void 176 cpu_topology_link(struct cpu_info *ci, struct cpu_info *ci2, enum cpu_rel rel) 177 { 178 struct cpu_info *ci3; 179 180 /* Walk to the end of the existing circular list and append. */ 181 for (ci3 = ci2;; ci3 = ci3->ci_sibling[rel]) { 182 ci3->ci_nsibling[rel]++; 183 if (ci3->ci_sibling[rel] == ci2) { 184 break; 185 } 186 } 187 ci->ci_sibling[rel] = ci2; 188 ci3->ci_sibling[rel] = ci; 189 ci->ci_nsibling[rel] = ci3->ci_nsibling[rel]; 190 } 191 192 /* 193 * Print out the topology lists. 194 */ 195 static void 196 cpu_topology_dump(void) 197 { 198 #ifdef DEBUG 199 CPU_INFO_ITERATOR cii; 200 struct cpu_info *ci, *ci2; 201 const char *names[] = { "core", "pkg", "1st" }; 202 enum cpu_rel rel; 203 int i; 204 205 CTASSERT(__arraycount(names) >= __arraycount(ci->ci_sibling)); 206 207 for (CPU_INFO_FOREACH(cii, ci)) { 208 if (cpu_topology_haveslow) 209 printf("%s ", ci->ci_is_slow ? "slow" : "fast"); 210 for (rel = 0; rel < __arraycount(ci->ci_sibling); rel++) { 211 printf("%s has %d %s siblings:", cpu_name(ci), 212 ci->ci_nsibling[rel], names[rel]); 213 ci2 = ci->ci_sibling[rel]; 214 i = 0; 215 do { 216 printf(" %s", cpu_name(ci2)); 217 ci2 = ci2->ci_sibling[rel]; 218 } while (++i < 64 && ci2 != ci->ci_sibling[rel]); 219 if (i == 64) { 220 printf(" GAVE UP"); 221 } 222 printf("\n"); 223 } 224 printf("%s first in package: %s\n", cpu_name(ci), 225 cpu_name(ci->ci_package1st)); 226 } 227 #endif /* DEBUG */ 228 } 229 230 /* 231 * Fake up topology info if we have none, or if what we got was bogus. 232 * Used early in boot, and by cpu_topology_fake(). 233 */ 234 static void 235 cpu_topology_fake1(struct cpu_info *ci) 236 { 237 enum cpu_rel rel; 238 239 for (rel = 0; rel < __arraycount(ci->ci_sibling); rel++) { 240 ci->ci_sibling[rel] = ci; 241 ci->ci_nsibling[rel] = 1; 242 } 243 if (!cpu_topology_present) { 244 ci->ci_package_id = cpu_index(ci); 245 } 246 ci->ci_schedstate.spc_flags |= 247 (SPCF_CORE1ST | SPCF_PACKAGE1ST | SPCF_1STCLASS); 248 ci->ci_package1st = ci; 249 if (!cpu_topology_haveslow) { 250 ci->ci_is_slow = false; 251 } 252 } 253 254 /* 255 * Fake up topology info if we have none, or if what we got was bogus. 256 * Don't override ci_package_id, etc, if cpu_topology_present is set. 257 * MD code also uses these. 258 */ 259 static void 260 cpu_topology_fake(void) 261 { 262 CPU_INFO_ITERATOR cii; 263 struct cpu_info *ci; 264 265 for (CPU_INFO_FOREACH(cii, ci)) { 266 cpu_topology_fake1(ci); 267 /* Undo (early boot) flag set so everything links OK. */ 268 ci->ci_schedstate.spc_flags &= 269 ~(SPCF_CORE1ST | SPCF_PACKAGE1ST | SPCF_1STCLASS); 270 } 271 } 272 273 /* 274 * Fix up basic CPU topology info. Right now that means attach each CPU to 275 * circular lists of its siblings in the same core, and in the same package. 276 */ 277 void 278 cpu_topology_init(void) 279 { 280 CPU_INFO_ITERATOR cii, cii2; 281 struct cpu_info *ci, *ci2, *ci3; 282 u_int minsmt, mincore; 283 284 if (!cpu_topology_present) { 285 cpu_topology_fake(); 286 goto linkit; 287 } 288 289 /* Find siblings in same core and package. */ 290 for (CPU_INFO_FOREACH(cii, ci)) { 291 ci->ci_schedstate.spc_flags &= 292 ~(SPCF_CORE1ST | SPCF_PACKAGE1ST | SPCF_1STCLASS); 293 for (CPU_INFO_FOREACH(cii2, ci2)) { 294 /* Avoid bad things happening. */ 295 if (ci2->ci_package_id == ci->ci_package_id && 296 ci2->ci_core_id == ci->ci_core_id && 297 ci2->ci_smt_id == ci->ci_smt_id && 298 ci2 != ci) { 299 #ifdef DEBUG 300 printf("cpu%u %p pkg %u core %u smt %u same as " 301 "cpu%u %p pkg %u core %u smt %u\n", 302 cpu_index(ci), ci, ci->ci_package_id, 303 ci->ci_core_id, ci->ci_smt_id, 304 cpu_index(ci2), ci2, ci2->ci_package_id, 305 ci2->ci_core_id, ci2->ci_smt_id); 306 #endif 307 printf("cpu_topology_init: info bogus, " 308 "faking it\n"); 309 cpu_topology_fake(); 310 goto linkit; 311 } 312 if (ci2 == ci || 313 ci2->ci_package_id != ci->ci_package_id) { 314 continue; 315 } 316 /* Find CPUs in the same core. */ 317 if (ci->ci_nsibling[CPUREL_CORE] == 1 && 318 ci->ci_core_id == ci2->ci_core_id) { 319 cpu_topology_link(ci, ci2, CPUREL_CORE); 320 } 321 /* Find CPUs in the same package. */ 322 if (ci->ci_nsibling[CPUREL_PACKAGE] == 1) { 323 cpu_topology_link(ci, ci2, CPUREL_PACKAGE); 324 } 325 if (ci->ci_nsibling[CPUREL_CORE] > 1 && 326 ci->ci_nsibling[CPUREL_PACKAGE] > 1) { 327 break; 328 } 329 } 330 } 331 332 linkit: 333 /* Identify lowest numbered SMT in each core. */ 334 for (CPU_INFO_FOREACH(cii, ci)) { 335 ci2 = ci3 = ci; 336 minsmt = ci->ci_smt_id; 337 do { 338 if (ci2->ci_smt_id < minsmt) { 339 ci3 = ci2; 340 minsmt = ci2->ci_smt_id; 341 } 342 ci2 = ci2->ci_sibling[CPUREL_CORE]; 343 } while (ci2 != ci); 344 ci3->ci_schedstate.spc_flags |= SPCF_CORE1ST; 345 } 346 347 /* Identify lowest numbered SMT in each package. */ 348 ci3 = NULL; 349 for (CPU_INFO_FOREACH(cii, ci)) { 350 if ((ci->ci_schedstate.spc_flags & SPCF_CORE1ST) == 0) { 351 continue; 352 } 353 ci2 = ci3 = ci; 354 mincore = ci->ci_core_id; 355 do { 356 if ((ci2->ci_schedstate.spc_flags & 357 SPCF_CORE1ST) != 0 && 358 ci2->ci_core_id < mincore) { 359 ci3 = ci2; 360 mincore = ci2->ci_core_id; 361 } 362 ci2 = ci2->ci_sibling[CPUREL_PACKAGE]; 363 } while (ci2 != ci); 364 365 if ((ci3->ci_schedstate.spc_flags & SPCF_PACKAGE1ST) != 0) { 366 /* Already identified - nothing more to do. */ 367 continue; 368 } 369 ci3->ci_schedstate.spc_flags |= SPCF_PACKAGE1ST; 370 371 /* Walk through all CPUs in package and point to first. */ 372 ci2 = ci3; 373 do { 374 ci2->ci_package1st = ci3; 375 ci2->ci_sibling[CPUREL_PACKAGE1ST] = ci3; 376 ci2 = ci2->ci_sibling[CPUREL_PACKAGE]; 377 } while (ci2 != ci3); 378 379 /* Now look for somebody else to link to. */ 380 for (CPU_INFO_FOREACH(cii2, ci2)) { 381 if ((ci2->ci_schedstate.spc_flags & SPCF_PACKAGE1ST) 382 != 0 && ci2 != ci3) { 383 cpu_topology_link(ci3, ci2, CPUREL_PACKAGE1ST); 384 break; 385 } 386 } 387 } 388 389 /* Walk through all packages, starting with value of ci3 from above. */ 390 KASSERT(ci3 != NULL); 391 ci = ci3; 392 do { 393 /* Walk through CPUs in the package and copy in PACKAGE1ST. */ 394 ci2 = ci; 395 do { 396 ci2->ci_sibling[CPUREL_PACKAGE1ST] = 397 ci->ci_sibling[CPUREL_PACKAGE1ST]; 398 ci2->ci_nsibling[CPUREL_PACKAGE1ST] = 399 ci->ci_nsibling[CPUREL_PACKAGE1ST]; 400 ci2 = ci2->ci_sibling[CPUREL_PACKAGE]; 401 } while (ci2 != ci); 402 ci = ci->ci_sibling[CPUREL_PACKAGE1ST]; 403 } while (ci != ci3); 404 405 if (cpu_topology_haveslow) { 406 /* 407 * For asymmetric systems where some CPUs are slower than 408 * others, mark first class CPUs for the scheduler. This 409 * conflicts with SMT right now so whinge if observed. 410 */ 411 if (curcpu()->ci_nsibling[CPUREL_CORE] > 1) { 412 printf("cpu_topology_init: asymmetric & SMT??\n"); 413 } 414 for (CPU_INFO_FOREACH(cii, ci)) { 415 if (!ci->ci_is_slow) { 416 ci->ci_schedstate.spc_flags |= SPCF_1STCLASS; 417 } 418 } 419 } else { 420 /* 421 * For any other configuration mark the 1st CPU in each 422 * core as a first class CPU. 423 */ 424 for (CPU_INFO_FOREACH(cii, ci)) { 425 if ((ci->ci_schedstate.spc_flags & SPCF_CORE1ST) != 0) { 426 ci->ci_schedstate.spc_flags |= SPCF_1STCLASS; 427 } 428 } 429 } 430 431 cpu_topology_dump(); 432 } 433 434 /* 435 * Adjust one count, for a counter that's NOT updated from interrupt 436 * context. Hardly worth making an inline due to preemption stuff. 437 */ 438 void 439 cpu_count(enum cpu_count idx, int64_t delta) 440 { 441 lwp_t *l = curlwp; 442 KPREEMPT_DISABLE(l); 443 l->l_cpu->ci_counts[idx] += delta; 444 KPREEMPT_ENABLE(l); 445 } 446 447 /* 448 * Fetch fresh sum total for all counts. Expensive - don't call often. 449 * 450 * If poll is true, the the caller is okay with with less recent values (but 451 * no more than 1/hz seconds old). Where this is called very often that 452 * should be the case. 453 * 454 * This should be reasonably quick so that any value collected get isn't 455 * totally out of whack, and it can also be called from interrupt context, 456 * so go to splvm() while summing the counters. It's tempting to use a spin 457 * mutex here but this routine is called from DDB. 458 */ 459 void 460 cpu_count_sync(bool poll) 461 { 462 CPU_INFO_ITERATOR cii; 463 struct cpu_info *ci; 464 int64_t sum[CPU_COUNT_MAX], *ptr; 465 static int lasttick; 466 int curtick, s; 467 enum cpu_count i; 468 469 KASSERT(sizeof(ci->ci_counts) == sizeof(cpu_counts)); 470 471 if (__predict_false(!mp_online)) { 472 memcpy(cpu_counts, curcpu()->ci_counts, sizeof(cpu_counts)); 473 return; 474 } 475 476 s = splvm(); 477 curtick = getticks(); 478 if (poll && atomic_load_acquire(&lasttick) == curtick) { 479 splx(s); 480 return; 481 } 482 memset(sum, 0, sizeof(sum)); 483 curcpu()->ci_counts[CPU_COUNT_SYNC]++; 484 for (CPU_INFO_FOREACH(cii, ci)) { 485 ptr = ci->ci_counts; 486 for (i = 0; i < CPU_COUNT_MAX; i += 8) { 487 sum[i+0] += ptr[i+0]; 488 sum[i+1] += ptr[i+1]; 489 sum[i+2] += ptr[i+2]; 490 sum[i+3] += ptr[i+3]; 491 sum[i+4] += ptr[i+4]; 492 sum[i+5] += ptr[i+5]; 493 sum[i+6] += ptr[i+6]; 494 sum[i+7] += ptr[i+7]; 495 } 496 KASSERT(i == CPU_COUNT_MAX); 497 } 498 memcpy(cpu_counts, sum, sizeof(cpu_counts)); 499 atomic_store_release(&lasttick, curtick); 500 splx(s); 501 } 502