1 /* $NetBSD: kern_clock.c,v 1.150 2023/07/07 12:34:50 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2000, 2004, 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * This code is derived from software contributed to The NetBSD Foundation 11 * by Charles M. Hannum. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 * POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 /*- 36 * Copyright (c) 1982, 1986, 1991, 1993 37 * The Regents of the University of California. All rights reserved. 38 * (c) UNIX System Laboratories, Inc. 39 * All or some portions of this file are derived from material licensed 40 * to the University of California by American Telephone and Telegraph 41 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 42 * the permission of UNIX System Laboratories, Inc. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. Neither the name of the University nor the names of its contributors 53 * may be used to endorse or promote products derived from this software 54 * without specific prior written permission. 55 * 56 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 57 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 58 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 59 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 60 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 61 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 62 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 63 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 64 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 65 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 66 * SUCH DAMAGE. 67 * 68 * @(#)kern_clock.c 8.5 (Berkeley) 1/21/94 69 */ 70 71 #include <sys/cdefs.h> 72 __KERNEL_RCSID(0, "$NetBSD: kern_clock.c,v 1.150 2023/07/07 12:34:50 riastradh Exp $"); 73 74 #ifdef _KERNEL_OPT 75 #include "opt_dtrace.h" 76 #include "opt_gprof.h" 77 #include "opt_heartbeat.h" 78 #include "opt_multiprocessor.h" 79 #endif 80 81 #include <sys/param.h> 82 #include <sys/systm.h> 83 #include <sys/callout.h> 84 #include <sys/kernel.h> 85 #include <sys/proc.h> 86 #include <sys/resourcevar.h> 87 #include <sys/signalvar.h> 88 #include <sys/sysctl.h> 89 #include <sys/timex.h> 90 #include <sys/sched.h> 91 #include <sys/time.h> 92 #include <sys/timetc.h> 93 #include <sys/cpu.h> 94 #include <sys/atomic.h> 95 #include <sys/rndsource.h> 96 #include <sys/heartbeat.h> 97 98 #ifdef GPROF 99 #include <sys/gmon.h> 100 #endif 101 102 #ifdef KDTRACE_HOOKS 103 #include <sys/dtrace_bsd.h> 104 #include <sys/cpu.h> 105 106 cyclic_clock_func_t cyclic_clock_func[MAXCPUS]; 107 #endif 108 109 static int sysctl_kern_clockrate(SYSCTLFN_PROTO); 110 111 /* 112 * Clock handling routines. 113 * 114 * This code is written to operate with two timers that run independently of 115 * each other. The main clock, running hz times per second, is used to keep 116 * track of real time. The second timer handles kernel and user profiling, 117 * and does resource use estimation. If the second timer is programmable, 118 * it is randomized to avoid aliasing between the two clocks. For example, 119 * the randomization prevents an adversary from always giving up the CPU 120 * just before its quantum expires. Otherwise, it would never accumulate 121 * CPU ticks. The mean frequency of the second timer is stathz. 122 * 123 * If no second timer exists, stathz will be zero; in this case we drive 124 * profiling and statistics off the main clock. This WILL NOT be accurate; 125 * do not do it unless absolutely necessary. 126 * 127 * The statistics clock may (or may not) be run at a higher rate while 128 * profiling. This profile clock runs at profhz. We require that profhz 129 * be an integral multiple of stathz. 130 * 131 * If the statistics clock is running fast, it must be divided by the ratio 132 * profhz/stathz for statistics. (For profiling, every tick counts.) 133 */ 134 135 int stathz; 136 int profhz; 137 int profsrc; 138 int schedhz; 139 int profprocs; 140 static int hardclock_ticks; 141 static int hardscheddiv; /* hard => sched divider (used if schedhz == 0) */ 142 static int psdiv; /* prof => stat divider */ 143 int psratio; /* ratio: prof / stat */ 144 145 struct clockrnd { 146 struct krndsource source; 147 unsigned needed; 148 }; 149 150 static struct clockrnd hardclockrnd __aligned(COHERENCY_UNIT); 151 static struct clockrnd statclockrnd __aligned(COHERENCY_UNIT); 152 153 static void 154 clockrnd_get(size_t needed, void *cookie) 155 { 156 struct clockrnd *C = cookie; 157 158 /* Start sampling. */ 159 atomic_store_relaxed(&C->needed, 2*NBBY*needed); 160 } 161 162 static void 163 clockrnd_sample(struct clockrnd *C) 164 { 165 struct cpu_info *ci = curcpu(); 166 167 /* If there's nothing needed right now, stop here. */ 168 if (__predict_true(atomic_load_relaxed(&C->needed) == 0)) 169 return; 170 171 /* 172 * If we're not the primary core of a package, we're probably 173 * driven by the same clock as the primary core, so don't 174 * bother. 175 */ 176 if (ci != ci->ci_package1st) 177 return; 178 179 /* Take a sample and enter it into the pool. */ 180 rnd_add_uint32(&C->source, 0); 181 182 /* 183 * On the primary CPU, count down. Using an atomic decrement 184 * here isn't really necessary -- on every platform we care 185 * about, stores to unsigned int are atomic, and the only other 186 * memory operation that could happen here is for another CPU 187 * to store a higher value for needed. But using an atomic 188 * decrement avoids giving the impression of data races, and is 189 * unlikely to hurt because only one CPU will ever be writing 190 * to the location. 191 */ 192 if (CPU_IS_PRIMARY(curcpu())) { 193 unsigned needed __diagused; 194 195 needed = atomic_dec_uint_nv(&C->needed); 196 KASSERT(needed != UINT_MAX); 197 } 198 } 199 200 static u_int get_intr_timecount(struct timecounter *); 201 202 static struct timecounter intr_timecounter = { 203 .tc_get_timecount = get_intr_timecount, 204 .tc_poll_pps = NULL, 205 .tc_counter_mask = ~0u, 206 .tc_frequency = 0, 207 .tc_name = "clockinterrupt", 208 /* quality - minimum implementation level for a clock */ 209 .tc_quality = 0, 210 .tc_priv = NULL, 211 }; 212 213 static u_int 214 get_intr_timecount(struct timecounter *tc) 215 { 216 217 return (u_int)getticks(); 218 } 219 220 int 221 getticks(void) 222 { 223 return atomic_load_relaxed(&hardclock_ticks); 224 } 225 226 /* 227 * Initialize clock frequencies and start both clocks running. 228 */ 229 void 230 initclocks(void) 231 { 232 static struct sysctllog *clog; 233 int i; 234 235 /* 236 * Set divisors to 1 (normal case) and let the machine-specific 237 * code do its bit. 238 */ 239 psdiv = 1; 240 241 /* 242 * Call cpu_initclocks() before registering the default 243 * timecounter, in case it needs to adjust hz. 244 */ 245 const int old_hz = hz; 246 cpu_initclocks(); 247 if (old_hz != hz) { 248 tick = 1000000 / hz; 249 tickadj = (240000 / (60 * hz)) ? (240000 / (60 * hz)) : 1; 250 } 251 252 /* 253 * provide minimum default time counter 254 * will only run at interrupt resolution 255 */ 256 intr_timecounter.tc_frequency = hz; 257 tc_init(&intr_timecounter); 258 259 /* 260 * Compute profhz and stathz, fix profhz if needed. 261 */ 262 i = stathz ? stathz : hz; 263 if (profhz == 0) 264 profhz = i; 265 psratio = profhz / i; 266 if (schedhz == 0) { 267 /* 16Hz is best */ 268 hardscheddiv = hz / 16; 269 if (hardscheddiv <= 0) 270 panic("hardscheddiv"); 271 } 272 273 sysctl_createv(&clog, 0, NULL, NULL, 274 CTLFLAG_PERMANENT, 275 CTLTYPE_STRUCT, "clockrate", 276 SYSCTL_DESCR("Kernel clock rates"), 277 sysctl_kern_clockrate, 0, NULL, 278 sizeof(struct clockinfo), 279 CTL_KERN, KERN_CLOCKRATE, CTL_EOL); 280 sysctl_createv(&clog, 0, NULL, NULL, 281 CTLFLAG_PERMANENT, 282 CTLTYPE_INT, "hardclock_ticks", 283 SYSCTL_DESCR("Number of hardclock ticks"), 284 NULL, 0, &hardclock_ticks, sizeof(hardclock_ticks), 285 CTL_KERN, KERN_HARDCLOCK_TICKS, CTL_EOL); 286 287 rndsource_setcb(&hardclockrnd.source, clockrnd_get, &hardclockrnd); 288 rnd_attach_source(&hardclockrnd.source, "hardclock", RND_TYPE_SKEW, 289 RND_FLAG_COLLECT_TIME|RND_FLAG_ESTIMATE_TIME|RND_FLAG_HASCB); 290 if (stathz) { 291 rndsource_setcb(&statclockrnd.source, clockrnd_get, 292 &statclockrnd); 293 rnd_attach_source(&statclockrnd.source, "statclock", 294 RND_TYPE_SKEW, 295 (RND_FLAG_COLLECT_TIME|RND_FLAG_ESTIMATE_TIME| 296 RND_FLAG_HASCB)); 297 } 298 } 299 300 /* 301 * The real-time timer, interrupting hz times per second. 302 */ 303 void 304 hardclock(struct clockframe *frame) 305 { 306 struct lwp *l; 307 struct cpu_info *ci; 308 309 clockrnd_sample(&hardclockrnd); 310 311 ci = curcpu(); 312 l = ci->ci_onproc; 313 314 ptimer_tick(l, CLKF_USERMODE(frame)); 315 316 /* 317 * If no separate statistics clock is available, run it from here. 318 */ 319 if (stathz == 0) 320 statclock(frame); 321 /* 322 * If no separate schedclock is provided, call it here 323 * at about 16 Hz. 324 */ 325 if (schedhz == 0) { 326 if ((int)(--ci->ci_schedstate.spc_schedticks) <= 0) { 327 schedclock(l); 328 ci->ci_schedstate.spc_schedticks = hardscheddiv; 329 } 330 } 331 if ((--ci->ci_schedstate.spc_ticks) <= 0) 332 sched_tick(ci); 333 334 if (CPU_IS_PRIMARY(ci)) { 335 atomic_store_relaxed(&hardclock_ticks, 336 atomic_load_relaxed(&hardclock_ticks) + 1); 337 tc_ticktock(); 338 } 339 340 #ifdef HEARTBEAT 341 /* 342 * Make sure the CPUs and timecounter are making progress. 343 */ 344 heartbeat(); 345 #endif 346 347 /* 348 * Update real-time timeout queue. 349 */ 350 callout_hardclock(); 351 } 352 353 /* 354 * Start profiling on a process. 355 * 356 * Kernel profiling passes proc0 which never exits and hence 357 * keeps the profile clock running constantly. 358 */ 359 void 360 startprofclock(struct proc *p) 361 { 362 363 KASSERT(mutex_owned(&p->p_stmutex)); 364 365 if ((p->p_stflag & PST_PROFIL) == 0) { 366 p->p_stflag |= PST_PROFIL; 367 /* 368 * This is only necessary if using the clock as the 369 * profiling source. 370 */ 371 if (++profprocs == 1 && stathz != 0) 372 psdiv = psratio; 373 } 374 } 375 376 /* 377 * Stop profiling on a process. 378 */ 379 void 380 stopprofclock(struct proc *p) 381 { 382 383 KASSERT(mutex_owned(&p->p_stmutex)); 384 385 if (p->p_stflag & PST_PROFIL) { 386 p->p_stflag &= ~PST_PROFIL; 387 /* 388 * This is only necessary if using the clock as the 389 * profiling source. 390 */ 391 if (--profprocs == 0 && stathz != 0) 392 psdiv = 1; 393 } 394 } 395 396 void 397 schedclock(struct lwp *l) 398 { 399 if ((l->l_flag & LW_IDLE) != 0) 400 return; 401 402 sched_schedclock(l); 403 } 404 405 /* 406 * Statistics clock. Grab profile sample, and if divider reaches 0, 407 * do process and kernel statistics. 408 */ 409 void 410 statclock(struct clockframe *frame) 411 { 412 #ifdef GPROF 413 struct gmonparam *g; 414 intptr_t i; 415 #endif 416 struct cpu_info *ci = curcpu(); 417 struct schedstate_percpu *spc = &ci->ci_schedstate; 418 struct proc *p; 419 struct lwp *l; 420 421 if (stathz) 422 clockrnd_sample(&statclockrnd); 423 424 /* 425 * Notice changes in divisor frequency, and adjust clock 426 * frequency accordingly. 427 */ 428 if (spc->spc_psdiv != psdiv) { 429 spc->spc_psdiv = psdiv; 430 spc->spc_pscnt = psdiv; 431 if (psdiv == 1) { 432 setstatclockrate(stathz); 433 } else { 434 setstatclockrate(profhz); 435 } 436 } 437 l = ci->ci_onproc; 438 if ((l->l_flag & LW_IDLE) != 0) { 439 /* 440 * don't account idle lwps as swapper. 441 */ 442 p = NULL; 443 } else { 444 p = l->l_proc; 445 mutex_spin_enter(&p->p_stmutex); 446 } 447 448 if (CLKF_USERMODE(frame)) { 449 KASSERT(p != NULL); 450 if ((p->p_stflag & PST_PROFIL) && profsrc == PROFSRC_CLOCK) 451 addupc_intr(l, CLKF_PC(frame)); 452 if (--spc->spc_pscnt > 0) { 453 mutex_spin_exit(&p->p_stmutex); 454 return; 455 } 456 457 /* 458 * Came from user mode; CPU was in user state. 459 * If this process is being profiled record the tick. 460 */ 461 p->p_uticks++; 462 if (p->p_nice > NZERO) 463 spc->spc_cp_time[CP_NICE]++; 464 else 465 spc->spc_cp_time[CP_USER]++; 466 } else { 467 #ifdef GPROF 468 /* 469 * Kernel statistics are just like addupc_intr, only easier. 470 */ 471 #if defined(MULTIPROCESSOR) && !defined(_RUMPKERNEL) 472 g = curcpu()->ci_gmon; 473 if (g != NULL && 474 profsrc == PROFSRC_CLOCK && g->state == GMON_PROF_ON) { 475 #else 476 g = &_gmonparam; 477 if (profsrc == PROFSRC_CLOCK && g->state == GMON_PROF_ON) { 478 #endif 479 i = CLKF_PC(frame) - g->lowpc; 480 if (i < g->textsize) { 481 i /= HISTFRACTION * sizeof(*g->kcount); 482 g->kcount[i]++; 483 } 484 } 485 #endif 486 #ifdef LWP_PC 487 if (p != NULL && profsrc == PROFSRC_CLOCK && 488 (p->p_stflag & PST_PROFIL)) { 489 addupc_intr(l, LWP_PC(l)); 490 } 491 #endif 492 if (--spc->spc_pscnt > 0) { 493 if (p != NULL) 494 mutex_spin_exit(&p->p_stmutex); 495 return; 496 } 497 /* 498 * Came from kernel mode, so we were: 499 * - handling an interrupt, 500 * - doing syscall or trap work on behalf of the current 501 * user process, or 502 * - spinning in the idle loop. 503 * Whichever it is, charge the time as appropriate. 504 * Note that we charge interrupts to the current process, 505 * regardless of whether they are ``for'' that process, 506 * so that we know how much of its real time was spent 507 * in ``non-process'' (i.e., interrupt) work. 508 */ 509 if (CLKF_INTR(frame) || (curlwp->l_pflag & LP_INTR) != 0) { 510 if (p != NULL) { 511 p->p_iticks++; 512 } 513 spc->spc_cp_time[CP_INTR]++; 514 } else if (p != NULL) { 515 p->p_sticks++; 516 spc->spc_cp_time[CP_SYS]++; 517 } else { 518 spc->spc_cp_time[CP_IDLE]++; 519 } 520 } 521 spc->spc_pscnt = psdiv; 522 523 if (p != NULL) { 524 atomic_inc_uint(&l->l_cpticks); 525 mutex_spin_exit(&p->p_stmutex); 526 } 527 528 #ifdef KDTRACE_HOOKS 529 cyclic_clock_func_t func = cyclic_clock_func[cpu_index(ci)]; 530 if (func) { 531 (*func)((struct clockframe *)frame); 532 } 533 #endif 534 } 535 536 /* 537 * sysctl helper routine for kern.clockrate. Assembles a struct on 538 * the fly to be returned to the caller. 539 */ 540 static int 541 sysctl_kern_clockrate(SYSCTLFN_ARGS) 542 { 543 struct clockinfo clkinfo; 544 struct sysctlnode node; 545 546 clkinfo.tick = tick; 547 clkinfo.tickadj = tickadj; 548 clkinfo.hz = hz; 549 clkinfo.profhz = profhz; 550 clkinfo.stathz = stathz ? stathz : hz; 551 552 node = *rnode; 553 node.sysctl_data = &clkinfo; 554 return (sysctl_lookup(SYSCTLFN_CALL(&node))); 555 } 556