1 /* $NetBSD: kern_clock.c,v 1.75 2001/05/06 13:46:34 simonb Exp $ */ 2 3 /*- 4 * Copyright (c) 2000 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the NetBSD 22 * Foundation, Inc. and its contributors. 23 * 4. Neither the name of The NetBSD Foundation nor the names of its 24 * contributors may be used to endorse or promote products derived 25 * from this software without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 29 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 30 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 31 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 34 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 35 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 36 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 37 * POSSIBILITY OF SUCH DAMAGE. 38 */ 39 40 /*- 41 * Copyright (c) 1982, 1986, 1991, 1993 42 * The Regents of the University of California. All rights reserved. 43 * (c) UNIX System Laboratories, Inc. 44 * All or some portions of this file are derived from material licensed 45 * to the University of California by American Telephone and Telegraph 46 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 47 * the permission of UNIX System Laboratories, Inc. 48 * 49 * Redistribution and use in source and binary forms, with or without 50 * modification, are permitted provided that the following conditions 51 * are met: 52 * 1. Redistributions of source code must retain the above copyright 53 * notice, this list of conditions and the following disclaimer. 54 * 2. Redistributions in binary form must reproduce the above copyright 55 * notice, this list of conditions and the following disclaimer in the 56 * documentation and/or other materials provided with the distribution. 57 * 3. All advertising materials mentioning features or use of this software 58 * must display the following acknowledgement: 59 * This product includes software developed by the University of 60 * California, Berkeley and its contributors. 61 * 4. Neither the name of the University nor the names of its contributors 62 * may be used to endorse or promote products derived from this software 63 * without specific prior written permission. 64 * 65 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 66 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 67 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 68 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 69 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 70 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 71 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 72 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 73 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 74 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 75 * SUCH DAMAGE. 76 * 77 * @(#)kern_clock.c 8.5 (Berkeley) 1/21/94 78 */ 79 80 #include "opt_ntp.h" 81 82 #include <sys/param.h> 83 #include <sys/systm.h> 84 #include <sys/dkstat.h> 85 #include <sys/callout.h> 86 #include <sys/kernel.h> 87 #include <sys/proc.h> 88 #include <sys/resourcevar.h> 89 #include <sys/signalvar.h> 90 #include <uvm/uvm_extern.h> 91 #include <sys/sysctl.h> 92 #include <sys/timex.h> 93 #include <sys/sched.h> 94 95 #include <machine/cpu.h> 96 #ifdef __HAVE_GENERIC_SOFT_INTERRUPTS 97 #include <machine/intr.h> 98 #endif 99 100 #ifdef GPROF 101 #include <sys/gmon.h> 102 #endif 103 104 /* 105 * Clock handling routines. 106 * 107 * This code is written to operate with two timers that run independently of 108 * each other. The main clock, running hz times per second, is used to keep 109 * track of real time. The second timer handles kernel and user profiling, 110 * and does resource use estimation. If the second timer is programmable, 111 * it is randomized to avoid aliasing between the two clocks. For example, 112 * the randomization prevents an adversary from always giving up the cpu 113 * just before its quantum expires. Otherwise, it would never accumulate 114 * cpu ticks. The mean frequency of the second timer is stathz. 115 * 116 * If no second timer exists, stathz will be zero; in this case we drive 117 * profiling and statistics off the main clock. This WILL NOT be accurate; 118 * do not do it unless absolutely necessary. 119 * 120 * The statistics clock may (or may not) be run at a higher rate while 121 * profiling. This profile clock runs at profhz. We require that profhz 122 * be an integral multiple of stathz. 123 * 124 * If the statistics clock is running fast, it must be divided by the ratio 125 * profhz/stathz for statistics. (For profiling, every tick counts.) 126 */ 127 128 #ifdef NTP /* NTP phase-locked loop in kernel */ 129 /* 130 * Phase/frequency-lock loop (PLL/FLL) definitions 131 * 132 * The following variables are read and set by the ntp_adjtime() system 133 * call. 134 * 135 * time_state shows the state of the system clock, with values defined 136 * in the timex.h header file. 137 * 138 * time_status shows the status of the system clock, with bits defined 139 * in the timex.h header file. 140 * 141 * time_offset is used by the PLL/FLL to adjust the system time in small 142 * increments. 143 * 144 * time_constant determines the bandwidth or "stiffness" of the PLL. 145 * 146 * time_tolerance determines maximum frequency error or tolerance of the 147 * CPU clock oscillator and is a property of the architecture; however, 148 * in principle it could change as result of the presence of external 149 * discipline signals, for instance. 150 * 151 * time_precision is usually equal to the kernel tick variable; however, 152 * in cases where a precision clock counter or external clock is 153 * available, the resolution can be much less than this and depend on 154 * whether the external clock is working or not. 155 * 156 * time_maxerror is initialized by a ntp_adjtime() call and increased by 157 * the kernel once each second to reflect the maximum error bound 158 * growth. 159 * 160 * time_esterror is set and read by the ntp_adjtime() call, but 161 * otherwise not used by the kernel. 162 */ 163 int time_state = TIME_OK; /* clock state */ 164 int time_status = STA_UNSYNC; /* clock status bits */ 165 long time_offset = 0; /* time offset (us) */ 166 long time_constant = 0; /* pll time constant */ 167 long time_tolerance = MAXFREQ; /* frequency tolerance (scaled ppm) */ 168 long time_precision = 1; /* clock precision (us) */ 169 long time_maxerror = MAXPHASE; /* maximum error (us) */ 170 long time_esterror = MAXPHASE; /* estimated error (us) */ 171 172 /* 173 * The following variables establish the state of the PLL/FLL and the 174 * residual time and frequency offset of the local clock. The scale 175 * factors are defined in the timex.h header file. 176 * 177 * time_phase and time_freq are the phase increment and the frequency 178 * increment, respectively, of the kernel time variable. 179 * 180 * time_freq is set via ntp_adjtime() from a value stored in a file when 181 * the synchronization daemon is first started. Its value is retrieved 182 * via ntp_adjtime() and written to the file about once per hour by the 183 * daemon. 184 * 185 * time_adj is the adjustment added to the value of tick at each timer 186 * interrupt and is recomputed from time_phase and time_freq at each 187 * seconds rollover. 188 * 189 * time_reftime is the second's portion of the system time at the last 190 * call to ntp_adjtime(). It is used to adjust the time_freq variable 191 * and to increase the time_maxerror as the time since last update 192 * increases. 193 */ 194 long time_phase = 0; /* phase offset (scaled us) */ 195 long time_freq = 0; /* frequency offset (scaled ppm) */ 196 long time_adj = 0; /* tick adjust (scaled 1 / hz) */ 197 long time_reftime = 0; /* time at last adjustment (s) */ 198 199 #ifdef PPS_SYNC 200 /* 201 * The following variables are used only if the kernel PPS discipline 202 * code is configured (PPS_SYNC). The scale factors are defined in the 203 * timex.h header file. 204 * 205 * pps_time contains the time at each calibration interval, as read by 206 * microtime(). pps_count counts the seconds of the calibration 207 * interval, the duration of which is nominally pps_shift in powers of 208 * two. 209 * 210 * pps_offset is the time offset produced by the time median filter 211 * pps_tf[], while pps_jitter is the dispersion (jitter) measured by 212 * this filter. 213 * 214 * pps_freq is the frequency offset produced by the frequency median 215 * filter pps_ff[], while pps_stabil is the dispersion (wander) measured 216 * by this filter. 217 * 218 * pps_usec is latched from a high resolution counter or external clock 219 * at pps_time. Here we want the hardware counter contents only, not the 220 * contents plus the time_tv.usec as usual. 221 * 222 * pps_valid counts the number of seconds since the last PPS update. It 223 * is used as a watchdog timer to disable the PPS discipline should the 224 * PPS signal be lost. 225 * 226 * pps_glitch counts the number of seconds since the beginning of an 227 * offset burst more than tick/2 from current nominal offset. It is used 228 * mainly to suppress error bursts due to priority conflicts between the 229 * PPS interrupt and timer interrupt. 230 * 231 * pps_intcnt counts the calibration intervals for use in the interval- 232 * adaptation algorithm. It's just too complicated for words. 233 */ 234 struct timeval pps_time; /* kernel time at last interval */ 235 long pps_tf[] = {0, 0, 0}; /* pps time offset median filter (us) */ 236 long pps_offset = 0; /* pps time offset (us) */ 237 long pps_jitter = MAXTIME; /* time dispersion (jitter) (us) */ 238 long pps_ff[] = {0, 0, 0}; /* pps frequency offset median filter */ 239 long pps_freq = 0; /* frequency offset (scaled ppm) */ 240 long pps_stabil = MAXFREQ; /* frequency dispersion (scaled ppm) */ 241 long pps_usec = 0; /* microsec counter at last interval */ 242 long pps_valid = PPS_VALID; /* pps signal watchdog counter */ 243 int pps_glitch = 0; /* pps signal glitch counter */ 244 int pps_count = 0; /* calibration interval counter (s) */ 245 int pps_shift = PPS_SHIFT; /* interval duration (s) (shift) */ 246 int pps_intcnt = 0; /* intervals at current duration */ 247 248 /* 249 * PPS signal quality monitors 250 * 251 * pps_jitcnt counts the seconds that have been discarded because the 252 * jitter measured by the time median filter exceeds the limit MAXTIME 253 * (100 us). 254 * 255 * pps_calcnt counts the frequency calibration intervals, which are 256 * variable from 4 s to 256 s. 257 * 258 * pps_errcnt counts the calibration intervals which have been discarded 259 * because the wander exceeds the limit MAXFREQ (100 ppm) or where the 260 * calibration interval jitter exceeds two ticks. 261 * 262 * pps_stbcnt counts the calibration intervals that have been discarded 263 * because the frequency wander exceeds the limit MAXFREQ / 4 (25 us). 264 */ 265 long pps_jitcnt = 0; /* jitter limit exceeded */ 266 long pps_calcnt = 0; /* calibration intervals */ 267 long pps_errcnt = 0; /* calibration errors */ 268 long pps_stbcnt = 0; /* stability limit exceeded */ 269 #endif /* PPS_SYNC */ 270 271 #ifdef EXT_CLOCK 272 /* 273 * External clock definitions 274 * 275 * The following definitions and declarations are used only if an 276 * external clock is configured on the system. 277 */ 278 #define CLOCK_INTERVAL 30 /* CPU clock update interval (s) */ 279 280 /* 281 * The clock_count variable is set to CLOCK_INTERVAL at each PPS 282 * interrupt and decremented once each second. 283 */ 284 int clock_count = 0; /* CPU clock counter */ 285 286 #ifdef HIGHBALL 287 /* 288 * The clock_offset and clock_cpu variables are used by the HIGHBALL 289 * interface. The clock_offset variable defines the offset between 290 * system time and the HIGBALL counters. The clock_cpu variable contains 291 * the offset between the system clock and the HIGHBALL clock for use in 292 * disciplining the kernel time variable. 293 */ 294 extern struct timeval clock_offset; /* Highball clock offset */ 295 long clock_cpu = 0; /* CPU clock adjust */ 296 #endif /* HIGHBALL */ 297 #endif /* EXT_CLOCK */ 298 #endif /* NTP */ 299 300 301 /* 302 * Bump a timeval by a small number of usec's. 303 */ 304 #define BUMPTIME(t, usec) { \ 305 volatile struct timeval *tp = (t); \ 306 long us; \ 307 \ 308 tp->tv_usec = us = tp->tv_usec + (usec); \ 309 if (us >= 1000000) { \ 310 tp->tv_usec = us - 1000000; \ 311 tp->tv_sec++; \ 312 } \ 313 } 314 315 int stathz; 316 int profhz; 317 int schedhz; 318 int profprocs; 319 int softclock_running; /* 1 => softclock() is running */ 320 static int psdiv; /* prof => stat divider */ 321 int psratio; /* ratio: prof / stat */ 322 int tickfix, tickfixinterval; /* used if tick not really integral */ 323 #ifndef NTP 324 static int tickfixcnt; /* accumulated fractional error */ 325 #else 326 int fixtick; /* used by NTP for same */ 327 int shifthz; 328 #endif 329 330 /* 331 * We might want ldd to load the both words from time at once. 332 * To succeed we need to be quadword aligned. 333 * The sparc already does that, and that it has worked so far is a fluke. 334 */ 335 volatile struct timeval time __attribute__((__aligned__(__alignof__(quad_t)))); 336 volatile struct timeval mono_time; 337 338 /* 339 * The callout mechanism is based on the work of Adam M. Costello and 340 * George Varghese, published in a technical report entitled "Redesigning 341 * the BSD Callout and Timer Facilities", and Justin Gibbs's subsequent 342 * integration into FreeBSD, modified for NetBSD by Jason R. Thorpe. 343 * 344 * The original work on the data structures used in this implementation 345 * was published by G. Varghese and A. Lauck in the paper "Hashed and 346 * Hierarchical Timing Wheels: Data Structures for the Efficient 347 * Implementation of a Timer Facility" in the Proceedings of the 11th 348 * ACM Annual Symposium on Operating System Principles, Austin, Texas, 349 * November 1987. 350 */ 351 struct callout_queue *callwheel; 352 int callwheelsize, callwheelbits, callwheelmask; 353 354 static struct callout *nextsoftcheck; /* next callout to be checked */ 355 356 #ifdef CALLWHEEL_STATS 357 int callwheel_collisions; /* number of hash collisions */ 358 int callwheel_maxlength; /* length of the longest hash chain */ 359 int *callwheel_sizes; /* per-bucket length count */ 360 u_int64_t callwheel_count; /* # callouts currently */ 361 u_int64_t callwheel_established; /* # callouts established */ 362 u_int64_t callwheel_fired; /* # callouts that fired */ 363 u_int64_t callwheel_disestablished; /* # callouts disestablished */ 364 u_int64_t callwheel_changed; /* # callouts changed */ 365 u_int64_t callwheel_softclocks; /* # times softclock() called */ 366 u_int64_t callwheel_softchecks; /* # checks per softclock() */ 367 u_int64_t callwheel_softempty; /* # empty buckets seen */ 368 #endif /* CALLWHEEL_STATS */ 369 370 /* 371 * This value indicates the number of consecutive callouts that 372 * will be checked before we allow interrupts to have a chance 373 * again. 374 */ 375 #ifndef MAX_SOFTCLOCK_STEPS 376 #define MAX_SOFTCLOCK_STEPS 100 377 #endif 378 379 struct simplelock callwheel_slock; 380 381 #define CALLWHEEL_LOCK(s) \ 382 do { \ 383 s = splclock(); \ 384 simple_lock(&callwheel_slock); \ 385 } while (0) 386 387 #define CALLWHEEL_UNLOCK(s) \ 388 do { \ 389 simple_unlock(&callwheel_slock); \ 390 splx(s); \ 391 } while (0) 392 393 static void callout_stop_locked(struct callout *); 394 395 /* 396 * These are both protected by callwheel_lock. 397 * XXX SHOULD BE STATIC!! 398 */ 399 u_int64_t hardclock_ticks, softclock_ticks; 400 401 #ifdef __HAVE_GENERIC_SOFT_INTERRUPTS 402 void softclock(void *); 403 void *softclock_si; 404 #endif 405 406 /* 407 * Initialize clock frequencies and start both clocks running. 408 */ 409 void 410 initclocks(void) 411 { 412 int i; 413 414 #ifdef __HAVE_GENERIC_SOFT_INTERRUPTS 415 softclock_si = softintr_establish(IPL_SOFTCLOCK, softclock, NULL); 416 if (softclock_si == NULL) 417 panic("initclocks: unable to register softclock intr"); 418 #endif 419 420 /* 421 * Set divisors to 1 (normal case) and let the machine-specific 422 * code do its bit. 423 */ 424 psdiv = 1; 425 cpu_initclocks(); 426 427 /* 428 * Compute profhz/stathz/rrticks, and fix profhz if needed. 429 */ 430 i = stathz ? stathz : hz; 431 if (profhz == 0) 432 profhz = i; 433 psratio = profhz / i; 434 rrticks = hz / 10; 435 436 #ifdef NTP 437 switch (hz) { 438 case 1: 439 shifthz = SHIFT_SCALE - 0; 440 break; 441 case 2: 442 shifthz = SHIFT_SCALE - 1; 443 break; 444 case 4: 445 shifthz = SHIFT_SCALE - 2; 446 break; 447 case 8: 448 shifthz = SHIFT_SCALE - 3; 449 break; 450 case 16: 451 shifthz = SHIFT_SCALE - 4; 452 break; 453 case 32: 454 shifthz = SHIFT_SCALE - 5; 455 break; 456 case 60: 457 case 64: 458 shifthz = SHIFT_SCALE - 6; 459 break; 460 case 96: 461 case 100: 462 case 128: 463 shifthz = SHIFT_SCALE - 7; 464 break; 465 case 256: 466 shifthz = SHIFT_SCALE - 8; 467 break; 468 case 512: 469 shifthz = SHIFT_SCALE - 9; 470 break; 471 case 1000: 472 case 1024: 473 shifthz = SHIFT_SCALE - 10; 474 break; 475 case 1200: 476 case 2048: 477 shifthz = SHIFT_SCALE - 11; 478 break; 479 case 4096: 480 shifthz = SHIFT_SCALE - 12; 481 break; 482 case 8192: 483 shifthz = SHIFT_SCALE - 13; 484 break; 485 case 16384: 486 shifthz = SHIFT_SCALE - 14; 487 break; 488 case 32768: 489 shifthz = SHIFT_SCALE - 15; 490 break; 491 case 65536: 492 shifthz = SHIFT_SCALE - 16; 493 break; 494 default: 495 panic("weird hz"); 496 } 497 if (fixtick == 0) { 498 /* 499 * Give MD code a chance to set this to a better 500 * value; but, if it doesn't, we should. 501 */ 502 fixtick = (1000000 - (hz*tick)); 503 } 504 #endif 505 } 506 507 /* 508 * The real-time timer, interrupting hz times per second. 509 */ 510 void 511 hardclock(struct clockframe *frame) 512 { 513 struct proc *p; 514 int delta; 515 extern int tickdelta; 516 extern long timedelta; 517 struct cpu_info *ci = curcpu(); 518 #ifdef NTP 519 int time_update; 520 int ltemp; 521 #endif 522 523 p = curproc; 524 if (p) { 525 struct pstats *pstats; 526 527 /* 528 * Run current process's virtual and profile time, as needed. 529 */ 530 pstats = p->p_stats; 531 if (CLKF_USERMODE(frame) && 532 timerisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) && 533 itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) 534 psignal(p, SIGVTALRM); 535 if (timerisset(&pstats->p_timer[ITIMER_PROF].it_value) && 536 itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) 537 psignal(p, SIGPROF); 538 } 539 540 /* 541 * If no separate statistics clock is available, run it from here. 542 */ 543 if (stathz == 0) 544 statclock(frame); 545 if ((--ci->ci_schedstate.spc_rrticks) <= 0) 546 roundrobin(ci); 547 548 #if defined(MULTIPROCESSOR) 549 /* 550 * If we are not the primary CPU, we're not allowed to do 551 * any more work. 552 */ 553 if (CPU_IS_PRIMARY(ci) == 0) 554 return; 555 #endif 556 557 /* 558 * Increment the time-of-day. The increment is normally just 559 * ``tick''. If the machine is one which has a clock frequency 560 * such that ``hz'' would not divide the second evenly into 561 * milliseconds, a periodic adjustment must be applied. Finally, 562 * if we are still adjusting the time (see adjtime()), 563 * ``tickdelta'' may also be added in. 564 */ 565 delta = tick; 566 567 #ifndef NTP 568 if (tickfix) { 569 tickfixcnt += tickfix; 570 if (tickfixcnt >= tickfixinterval) { 571 delta++; 572 tickfixcnt -= tickfixinterval; 573 } 574 } 575 #endif /* !NTP */ 576 /* Imprecise 4bsd adjtime() handling */ 577 if (timedelta != 0) { 578 delta += tickdelta; 579 timedelta -= tickdelta; 580 } 581 582 #ifdef notyet 583 microset(); 584 #endif 585 586 #ifndef NTP 587 BUMPTIME(&time, delta); /* XXX Now done using NTP code below */ 588 #endif 589 BUMPTIME(&mono_time, delta); 590 591 #ifdef NTP 592 time_update = delta; 593 594 /* 595 * Compute the phase adjustment. If the low-order bits 596 * (time_phase) of the update overflow, bump the high-order bits 597 * (time_update). 598 */ 599 time_phase += time_adj; 600 if (time_phase <= -FINEUSEC) { 601 ltemp = -time_phase >> SHIFT_SCALE; 602 time_phase += ltemp << SHIFT_SCALE; 603 time_update -= ltemp; 604 } else if (time_phase >= FINEUSEC) { 605 ltemp = time_phase >> SHIFT_SCALE; 606 time_phase -= ltemp << SHIFT_SCALE; 607 time_update += ltemp; 608 } 609 610 #ifdef HIGHBALL 611 /* 612 * If the HIGHBALL board is installed, we need to adjust the 613 * external clock offset in order to close the hardware feedback 614 * loop. This will adjust the external clock phase and frequency 615 * in small amounts. The additional phase noise and frequency 616 * wander this causes should be minimal. We also need to 617 * discipline the kernel time variable, since the PLL is used to 618 * discipline the external clock. If the Highball board is not 619 * present, we discipline kernel time with the PLL as usual. We 620 * assume that the external clock phase adjustment (time_update) 621 * and kernel phase adjustment (clock_cpu) are less than the 622 * value of tick. 623 */ 624 clock_offset.tv_usec += time_update; 625 if (clock_offset.tv_usec >= 1000000) { 626 clock_offset.tv_sec++; 627 clock_offset.tv_usec -= 1000000; 628 } 629 if (clock_offset.tv_usec < 0) { 630 clock_offset.tv_sec--; 631 clock_offset.tv_usec += 1000000; 632 } 633 time.tv_usec += clock_cpu; 634 clock_cpu = 0; 635 #else 636 time.tv_usec += time_update; 637 #endif /* HIGHBALL */ 638 639 /* 640 * On rollover of the second the phase adjustment to be used for 641 * the next second is calculated. Also, the maximum error is 642 * increased by the tolerance. If the PPS frequency discipline 643 * code is present, the phase is increased to compensate for the 644 * CPU clock oscillator frequency error. 645 * 646 * On a 32-bit machine and given parameters in the timex.h 647 * header file, the maximum phase adjustment is +-512 ms and 648 * maximum frequency offset is a tad less than) +-512 ppm. On a 649 * 64-bit machine, you shouldn't need to ask. 650 */ 651 if (time.tv_usec >= 1000000) { 652 time.tv_usec -= 1000000; 653 time.tv_sec++; 654 time_maxerror += time_tolerance >> SHIFT_USEC; 655 656 /* 657 * Leap second processing. If in leap-insert state at 658 * the end of the day, the system clock is set back one 659 * second; if in leap-delete state, the system clock is 660 * set ahead one second. The microtime() routine or 661 * external clock driver will insure that reported time 662 * is always monotonic. The ugly divides should be 663 * replaced. 664 */ 665 switch (time_state) { 666 case TIME_OK: 667 if (time_status & STA_INS) 668 time_state = TIME_INS; 669 else if (time_status & STA_DEL) 670 time_state = TIME_DEL; 671 break; 672 673 case TIME_INS: 674 if (time.tv_sec % 86400 == 0) { 675 time.tv_sec--; 676 time_state = TIME_OOP; 677 } 678 break; 679 680 case TIME_DEL: 681 if ((time.tv_sec + 1) % 86400 == 0) { 682 time.tv_sec++; 683 time_state = TIME_WAIT; 684 } 685 break; 686 687 case TIME_OOP: 688 time_state = TIME_WAIT; 689 break; 690 691 case TIME_WAIT: 692 if (!(time_status & (STA_INS | STA_DEL))) 693 time_state = TIME_OK; 694 break; 695 } 696 697 /* 698 * Compute the phase adjustment for the next second. In 699 * PLL mode, the offset is reduced by a fixed factor 700 * times the time constant. In FLL mode the offset is 701 * used directly. In either mode, the maximum phase 702 * adjustment for each second is clamped so as to spread 703 * the adjustment over not more than the number of 704 * seconds between updates. 705 */ 706 if (time_offset < 0) { 707 ltemp = -time_offset; 708 if (!(time_status & STA_FLL)) 709 ltemp >>= SHIFT_KG + time_constant; 710 if (ltemp > (MAXPHASE / MINSEC) << SHIFT_UPDATE) 711 ltemp = (MAXPHASE / MINSEC) << 712 SHIFT_UPDATE; 713 time_offset += ltemp; 714 time_adj = -ltemp << (shifthz - SHIFT_UPDATE); 715 } else if (time_offset > 0) { 716 ltemp = time_offset; 717 if (!(time_status & STA_FLL)) 718 ltemp >>= SHIFT_KG + time_constant; 719 if (ltemp > (MAXPHASE / MINSEC) << SHIFT_UPDATE) 720 ltemp = (MAXPHASE / MINSEC) << 721 SHIFT_UPDATE; 722 time_offset -= ltemp; 723 time_adj = ltemp << (shifthz - SHIFT_UPDATE); 724 } else 725 time_adj = 0; 726 727 /* 728 * Compute the frequency estimate and additional phase 729 * adjustment due to frequency error for the next 730 * second. When the PPS signal is engaged, gnaw on the 731 * watchdog counter and update the frequency computed by 732 * the pll and the PPS signal. 733 */ 734 #ifdef PPS_SYNC 735 pps_valid++; 736 if (pps_valid == PPS_VALID) { 737 pps_jitter = MAXTIME; 738 pps_stabil = MAXFREQ; 739 time_status &= ~(STA_PPSSIGNAL | STA_PPSJITTER | 740 STA_PPSWANDER | STA_PPSERROR); 741 } 742 ltemp = time_freq + pps_freq; 743 #else 744 ltemp = time_freq; 745 #endif /* PPS_SYNC */ 746 747 if (ltemp < 0) 748 time_adj -= -ltemp >> (SHIFT_USEC - shifthz); 749 else 750 time_adj += ltemp >> (SHIFT_USEC - shifthz); 751 time_adj += (long)fixtick << shifthz; 752 753 /* 754 * When the CPU clock oscillator frequency is not a 755 * power of 2 in Hz, shifthz is only an approximate 756 * scale factor. 757 * 758 * To determine the adjustment, you can do the following: 759 * bc -q 760 * scale=24 761 * obase=2 762 * idealhz/realhz 763 * where `idealhz' is the next higher power of 2, and `realhz' 764 * is the actual value. You may need to factor this result 765 * into a sequence of 2 multipliers to get better precision. 766 * 767 * Likewise, the error can be calculated with (e.g. for 100Hz): 768 * bc -q 769 * scale=24 770 * ((1+2^-2+2^-5)*(1-2^-10)*realhz-idealhz)/idealhz 771 * (and then multiply by 1000000 to get ppm). 772 */ 773 switch (hz) { 774 case 60: 775 /* A factor of 1.000100010001 gives about 15ppm 776 error. */ 777 if (time_adj < 0) { 778 time_adj -= (-time_adj >> 4); 779 time_adj -= (-time_adj >> 8); 780 } else { 781 time_adj += (time_adj >> 4); 782 time_adj += (time_adj >> 8); 783 } 784 break; 785 786 case 96: 787 /* A factor of 1.0101010101 gives about 244ppm error. */ 788 if (time_adj < 0) { 789 time_adj -= (-time_adj >> 2); 790 time_adj -= (-time_adj >> 4) + (-time_adj >> 8); 791 } else { 792 time_adj += (time_adj >> 2); 793 time_adj += (time_adj >> 4) + (time_adj >> 8); 794 } 795 break; 796 797 case 100: 798 /* A factor of 1.010001111010111 gives about 1ppm 799 error. */ 800 if (time_adj < 0) { 801 time_adj -= (-time_adj >> 2) + (-time_adj >> 5); 802 time_adj += (-time_adj >> 10); 803 } else { 804 time_adj += (time_adj >> 2) + (time_adj >> 5); 805 time_adj -= (time_adj >> 10); 806 } 807 break; 808 809 case 1000: 810 /* A factor of 1.000001100010100001 gives about 50ppm 811 error. */ 812 if (time_adj < 0) { 813 time_adj -= (-time_adj >> 6) + (-time_adj >> 11); 814 time_adj -= (-time_adj >> 7); 815 } else { 816 time_adj += (time_adj >> 6) + (time_adj >> 11); 817 time_adj += (time_adj >> 7); 818 } 819 break; 820 821 case 1200: 822 /* A factor of 1.1011010011100001 gives about 64ppm 823 error. */ 824 if (time_adj < 0) { 825 time_adj -= (-time_adj >> 1) + (-time_adj >> 6); 826 time_adj -= (-time_adj >> 3) + (-time_adj >> 10); 827 } else { 828 time_adj += (time_adj >> 1) + (time_adj >> 6); 829 time_adj += (time_adj >> 3) + (time_adj >> 10); 830 } 831 break; 832 } 833 834 #ifdef EXT_CLOCK 835 /* 836 * If an external clock is present, it is necessary to 837 * discipline the kernel time variable anyway, since not 838 * all system components use the microtime() interface. 839 * Here, the time offset between the external clock and 840 * kernel time variable is computed every so often. 841 */ 842 clock_count++; 843 if (clock_count > CLOCK_INTERVAL) { 844 clock_count = 0; 845 microtime(&clock_ext); 846 delta.tv_sec = clock_ext.tv_sec - time.tv_sec; 847 delta.tv_usec = clock_ext.tv_usec - 848 time.tv_usec; 849 if (delta.tv_usec < 0) 850 delta.tv_sec--; 851 if (delta.tv_usec >= 500000) { 852 delta.tv_usec -= 1000000; 853 delta.tv_sec++; 854 } 855 if (delta.tv_usec < -500000) { 856 delta.tv_usec += 1000000; 857 delta.tv_sec--; 858 } 859 if (delta.tv_sec > 0 || (delta.tv_sec == 0 && 860 delta.tv_usec > MAXPHASE) || 861 delta.tv_sec < -1 || (delta.tv_sec == -1 && 862 delta.tv_usec < -MAXPHASE)) { 863 time = clock_ext; 864 delta.tv_sec = 0; 865 delta.tv_usec = 0; 866 } 867 #ifdef HIGHBALL 868 clock_cpu = delta.tv_usec; 869 #else /* HIGHBALL */ 870 hardupdate(delta.tv_usec); 871 #endif /* HIGHBALL */ 872 } 873 #endif /* EXT_CLOCK */ 874 } 875 876 #endif /* NTP */ 877 878 /* 879 * Process callouts at a very low cpu priority, so we don't keep the 880 * relatively high clock interrupt priority any longer than necessary. 881 */ 882 simple_lock(&callwheel_slock); /* already at splclock() */ 883 hardclock_ticks++; 884 if (TAILQ_FIRST(&callwheel[hardclock_ticks & callwheelmask]) != NULL) { 885 simple_unlock(&callwheel_slock); 886 if (CLKF_BASEPRI(frame)) { 887 /* 888 * Save the overhead of a software interrupt; 889 * it will happen as soon as we return, so do 890 * it now. 891 * 892 * NOTE: If we're at ``base priority'', softclock() 893 * was not already running. 894 */ 895 spllowersoftclock(); 896 KERNEL_LOCK(LK_CANRECURSE|LK_EXCLUSIVE); 897 softclock(NULL); 898 KERNEL_UNLOCK(); 899 } else { 900 #ifdef __HAVE_GENERIC_SOFT_INTERRUPTS 901 softintr_schedule(softclock_si); 902 #else 903 setsoftclock(); 904 #endif 905 } 906 return; 907 } else if (softclock_running == 0 && 908 (softclock_ticks + 1) == hardclock_ticks) { 909 softclock_ticks++; 910 } 911 simple_unlock(&callwheel_slock); 912 } 913 914 /* 915 * Software (low priority) clock interrupt. 916 * Run periodic events from timeout queue. 917 */ 918 /*ARGSUSED*/ 919 void 920 softclock(void *v) 921 { 922 struct callout_queue *bucket; 923 struct callout *c; 924 void (*func)(void *); 925 void *arg; 926 int s, idx; 927 int steps = 0; 928 929 CALLWHEEL_LOCK(s); 930 931 softclock_running = 1; 932 933 #ifdef CALLWHEEL_STATS 934 callwheel_softclocks++; 935 #endif 936 937 while (softclock_ticks != hardclock_ticks) { 938 softclock_ticks++; 939 idx = (int)(softclock_ticks & callwheelmask); 940 bucket = &callwheel[idx]; 941 c = TAILQ_FIRST(bucket); 942 #ifdef CALLWHEEL_STATS 943 if (c == NULL) 944 callwheel_softempty++; 945 #endif 946 while (c != NULL) { 947 #ifdef CALLWHEEL_STATS 948 callwheel_softchecks++; 949 #endif 950 if (c->c_time != softclock_ticks) { 951 c = TAILQ_NEXT(c, c_link); 952 if (++steps >= MAX_SOFTCLOCK_STEPS) { 953 nextsoftcheck = c; 954 /* Give interrupts a chance. */ 955 CALLWHEEL_UNLOCK(s); 956 CALLWHEEL_LOCK(s); 957 c = nextsoftcheck; 958 steps = 0; 959 } 960 } else { 961 nextsoftcheck = TAILQ_NEXT(c, c_link); 962 TAILQ_REMOVE(bucket, c, c_link); 963 #ifdef CALLWHEEL_STATS 964 callwheel_sizes[idx]--; 965 callwheel_fired++; 966 callwheel_count--; 967 #endif 968 func = c->c_func; 969 arg = c->c_arg; 970 c->c_func = NULL; 971 c->c_flags &= ~CALLOUT_PENDING; 972 CALLWHEEL_UNLOCK(s); 973 (*func)(arg); 974 CALLWHEEL_LOCK(s); 975 steps = 0; 976 c = nextsoftcheck; 977 } 978 } 979 } 980 nextsoftcheck = NULL; 981 softclock_running = 0; 982 CALLWHEEL_UNLOCK(s); 983 } 984 985 /* 986 * callout_setsize: 987 * 988 * Determine how many callwheels are necessary and 989 * set hash mask. Called from allocsys(). 990 */ 991 void 992 callout_setsize(void) 993 { 994 995 for (callwheelsize = 1; callwheelsize < ncallout; callwheelsize <<= 1) 996 /* loop */ ; 997 callwheelmask = callwheelsize - 1; 998 } 999 1000 /* 1001 * callout_startup: 1002 * 1003 * Initialize the callwheel buckets. 1004 */ 1005 void 1006 callout_startup(void) 1007 { 1008 int i; 1009 1010 for (i = 0; i < callwheelsize; i++) 1011 TAILQ_INIT(&callwheel[i]); 1012 1013 simple_lock_init(&callwheel_slock); 1014 } 1015 1016 /* 1017 * callout_init: 1018 * 1019 * Initialize a callout structure so that it can be used 1020 * by callout_reset() and callout_stop(). 1021 */ 1022 void 1023 callout_init(struct callout *c) 1024 { 1025 1026 memset(c, 0, sizeof(*c)); 1027 } 1028 1029 /* 1030 * callout_reset: 1031 * 1032 * Establish or change a timeout. 1033 */ 1034 void 1035 callout_reset(struct callout *c, int ticks, void (*func)(void *), void *arg) 1036 { 1037 struct callout_queue *bucket; 1038 int s; 1039 1040 if (ticks <= 0) 1041 ticks = 1; 1042 1043 CALLWHEEL_LOCK(s); 1044 1045 /* 1046 * If this callout's timer is already running, cancel it 1047 * before we modify it. 1048 */ 1049 if (c->c_flags & CALLOUT_PENDING) { 1050 callout_stop_locked(c); /* Already locked */ 1051 #ifdef CALLWHEEL_STATS 1052 callwheel_changed++; 1053 #endif 1054 } 1055 1056 c->c_arg = arg; 1057 c->c_func = func; 1058 c->c_flags = CALLOUT_ACTIVE | CALLOUT_PENDING; 1059 c->c_time = hardclock_ticks + ticks; 1060 1061 bucket = &callwheel[c->c_time & callwheelmask]; 1062 1063 #ifdef CALLWHEEL_STATS 1064 if (TAILQ_FIRST(bucket) != NULL) 1065 callwheel_collisions++; 1066 #endif 1067 1068 TAILQ_INSERT_TAIL(bucket, c, c_link); 1069 1070 #ifdef CALLWHEEL_STATS 1071 callwheel_count++; 1072 callwheel_established++; 1073 if (++callwheel_sizes[c->c_time & callwheelmask] > callwheel_maxlength) 1074 callwheel_maxlength = 1075 callwheel_sizes[c->c_time & callwheelmask]; 1076 #endif 1077 1078 CALLWHEEL_UNLOCK(s); 1079 } 1080 1081 /* 1082 * callout_stop_locked: 1083 * 1084 * Disestablish a timeout. Callwheel is locked. 1085 */ 1086 static void 1087 callout_stop_locked(struct callout *c) 1088 { 1089 1090 /* 1091 * Don't attempt to delete a callout that's not on the queue. 1092 */ 1093 if ((c->c_flags & CALLOUT_PENDING) == 0) { 1094 c->c_flags &= ~CALLOUT_ACTIVE; 1095 return; 1096 } 1097 1098 c->c_flags &= ~(CALLOUT_ACTIVE | CALLOUT_PENDING); 1099 1100 if (nextsoftcheck == c) 1101 nextsoftcheck = TAILQ_NEXT(c, c_link); 1102 1103 TAILQ_REMOVE(&callwheel[c->c_time & callwheelmask], c, c_link); 1104 #ifdef CALLWHEEL_STATS 1105 callwheel_count--; 1106 callwheel_disestablished++; 1107 callwheel_sizes[c->c_time & callwheelmask]--; 1108 #endif 1109 1110 c->c_func = NULL; 1111 } 1112 1113 /* 1114 * callout_stop: 1115 * 1116 * Disestablish a timeout. Callwheel is unlocked. This is 1117 * the standard entry point. 1118 */ 1119 void 1120 callout_stop(struct callout *c) 1121 { 1122 int s; 1123 1124 CALLWHEEL_LOCK(s); 1125 callout_stop_locked(c); 1126 CALLWHEEL_UNLOCK(s); 1127 } 1128 1129 #ifdef CALLWHEEL_STATS 1130 /* 1131 * callout_showstats: 1132 * 1133 * Display callout statistics. Call it from DDB. 1134 */ 1135 void 1136 callout_showstats(void) 1137 { 1138 u_int64_t curticks; 1139 int s; 1140 1141 s = splclock(); 1142 curticks = softclock_ticks; 1143 splx(s); 1144 1145 printf("Callwheel statistics:\n"); 1146 printf("\tCallouts currently queued: %llu\n", callwheel_count); 1147 printf("\tCallouts established: %llu\n", callwheel_established); 1148 printf("\tCallouts disestablished: %llu\n", callwheel_disestablished); 1149 if (callwheel_changed != 0) 1150 printf("\t\tOf those, %llu were changes\n", callwheel_changed); 1151 printf("\tCallouts that fired: %llu\n", callwheel_fired); 1152 printf("\tNumber of buckets: %d\n", callwheelsize); 1153 printf("\tNumber of hash collisions: %d\n", callwheel_collisions); 1154 printf("\tMaximum hash chain length: %d\n", callwheel_maxlength); 1155 printf("\tSoftclocks: %llu, Softchecks: %llu\n", 1156 callwheel_softclocks, callwheel_softchecks); 1157 printf("\t\tEmpty buckets seen: %llu\n", callwheel_softempty); 1158 } 1159 #endif 1160 1161 /* 1162 * Compute number of hz until specified time. Used to compute second 1163 * argument to callout_reset() from an absolute time. 1164 */ 1165 int 1166 hzto(struct timeval *tv) 1167 { 1168 unsigned long ticks; 1169 long sec, usec; 1170 int s; 1171 1172 /* 1173 * If the number of usecs in the whole seconds part of the time 1174 * difference fits in a long, then the total number of usecs will 1175 * fit in an unsigned long. Compute the total and convert it to 1176 * ticks, rounding up and adding 1 to allow for the current tick 1177 * to expire. Rounding also depends on unsigned long arithmetic 1178 * to avoid overflow. 1179 * 1180 * Otherwise, if the number of ticks in the whole seconds part of 1181 * the time difference fits in a long, then convert the parts to 1182 * ticks separately and add, using similar rounding methods and 1183 * overflow avoidance. This method would work in the previous 1184 * case, but it is slightly slower and assume that hz is integral. 1185 * 1186 * Otherwise, round the time difference down to the maximum 1187 * representable value. 1188 * 1189 * If ints are 32-bit, then the maximum value for any timeout in 1190 * 10ms ticks is 248 days. 1191 */ 1192 s = splclock(); 1193 sec = tv->tv_sec - time.tv_sec; 1194 usec = tv->tv_usec - time.tv_usec; 1195 splx(s); 1196 1197 if (usec < 0) { 1198 sec--; 1199 usec += 1000000; 1200 } 1201 1202 if (sec < 0 || (sec == 0 && usec <= 0)) { 1203 /* 1204 * Would expire now or in the past. Return 0 ticks. 1205 * This is different from the legacy hzto() interface, 1206 * and callers need to check for it. 1207 */ 1208 ticks = 0; 1209 } else if (sec <= (LONG_MAX / 1000000)) 1210 ticks = (((sec * 1000000) + (unsigned long)usec + (tick - 1)) 1211 / tick) + 1; 1212 else if (sec <= (LONG_MAX / hz)) 1213 ticks = (sec * hz) + 1214 (((unsigned long)usec + (tick - 1)) / tick) + 1; 1215 else 1216 ticks = LONG_MAX; 1217 1218 if (ticks > INT_MAX) 1219 ticks = INT_MAX; 1220 1221 return ((int)ticks); 1222 } 1223 1224 /* 1225 * Start profiling on a process. 1226 * 1227 * Kernel profiling passes proc0 which never exits and hence 1228 * keeps the profile clock running constantly. 1229 */ 1230 void 1231 startprofclock(struct proc *p) 1232 { 1233 1234 if ((p->p_flag & P_PROFIL) == 0) { 1235 p->p_flag |= P_PROFIL; 1236 if (++profprocs == 1 && stathz != 0) 1237 psdiv = psratio; 1238 } 1239 } 1240 1241 /* 1242 * Stop profiling on a process. 1243 */ 1244 void 1245 stopprofclock(struct proc *p) 1246 { 1247 1248 if (p->p_flag & P_PROFIL) { 1249 p->p_flag &= ~P_PROFIL; 1250 if (--profprocs == 0 && stathz != 0) 1251 psdiv = 1; 1252 } 1253 } 1254 1255 /* 1256 * Statistics clock. Grab profile sample, and if divider reaches 0, 1257 * do process and kernel statistics. 1258 */ 1259 void 1260 statclock(struct clockframe *frame) 1261 { 1262 #ifdef GPROF 1263 struct gmonparam *g; 1264 intptr_t i; 1265 #endif 1266 struct cpu_info *ci = curcpu(); 1267 struct schedstate_percpu *spc = &ci->ci_schedstate; 1268 struct proc *p; 1269 1270 /* 1271 * Notice changes in divisor frequency, and adjust clock 1272 * frequency accordingly. 1273 */ 1274 if (spc->spc_psdiv != psdiv) { 1275 spc->spc_psdiv = psdiv; 1276 spc->spc_pscnt = psdiv; 1277 if (psdiv == 1) { 1278 setstatclockrate(stathz); 1279 } else { 1280 setstatclockrate(profhz); 1281 } 1282 } 1283 p = curproc; 1284 if (CLKF_USERMODE(frame)) { 1285 if (p->p_flag & P_PROFIL) 1286 addupc_intr(p, CLKF_PC(frame)); 1287 if (--spc->spc_pscnt > 0) 1288 return; 1289 /* 1290 * Came from user mode; CPU was in user state. 1291 * If this process is being profiled record the tick. 1292 */ 1293 p->p_uticks++; 1294 if (p->p_nice > NZERO) 1295 spc->spc_cp_time[CP_NICE]++; 1296 else 1297 spc->spc_cp_time[CP_USER]++; 1298 } else { 1299 #ifdef GPROF 1300 /* 1301 * Kernel statistics are just like addupc_intr, only easier. 1302 */ 1303 g = &_gmonparam; 1304 if (g->state == GMON_PROF_ON) { 1305 i = CLKF_PC(frame) - g->lowpc; 1306 if (i < g->textsize) { 1307 i /= HISTFRACTION * sizeof(*g->kcount); 1308 g->kcount[i]++; 1309 } 1310 } 1311 #endif 1312 #ifdef PROC_PC 1313 if (p && p->p_flag & P_PROFIL) 1314 addupc_intr(p, PROC_PC(p)); 1315 #endif 1316 if (--spc->spc_pscnt > 0) 1317 return; 1318 /* 1319 * Came from kernel mode, so we were: 1320 * - handling an interrupt, 1321 * - doing syscall or trap work on behalf of the current 1322 * user process, or 1323 * - spinning in the idle loop. 1324 * Whichever it is, charge the time as appropriate. 1325 * Note that we charge interrupts to the current process, 1326 * regardless of whether they are ``for'' that process, 1327 * so that we know how much of its real time was spent 1328 * in ``non-process'' (i.e., interrupt) work. 1329 */ 1330 if (CLKF_INTR(frame)) { 1331 if (p != NULL) 1332 p->p_iticks++; 1333 spc->spc_cp_time[CP_INTR]++; 1334 } else if (p != NULL) { 1335 p->p_sticks++; 1336 spc->spc_cp_time[CP_SYS]++; 1337 } else 1338 spc->spc_cp_time[CP_IDLE]++; 1339 } 1340 spc->spc_pscnt = psdiv; 1341 1342 if (p != NULL) { 1343 ++p->p_cpticks; 1344 /* 1345 * If no separate schedclock is provided, call it here 1346 * at ~~12-25 Hz, ~~16 Hz is best 1347 */ 1348 if (schedhz == 0) 1349 if ((++ci->ci_schedstate.spc_schedticks & 3) == 0) 1350 schedclock(p); 1351 } 1352 } 1353 1354 1355 #ifdef NTP /* NTP phase-locked loop in kernel */ 1356 1357 /* 1358 * hardupdate() - local clock update 1359 * 1360 * This routine is called by ntp_adjtime() to update the local clock 1361 * phase and frequency. The implementation is of an adaptive-parameter, 1362 * hybrid phase/frequency-lock loop (PLL/FLL). The routine computes new 1363 * time and frequency offset estimates for each call. If the kernel PPS 1364 * discipline code is configured (PPS_SYNC), the PPS signal itself 1365 * determines the new time offset, instead of the calling argument. 1366 * Presumably, calls to ntp_adjtime() occur only when the caller 1367 * believes the local clock is valid within some bound (+-128 ms with 1368 * NTP). If the caller's time is far different than the PPS time, an 1369 * argument will ensue, and it's not clear who will lose. 1370 * 1371 * For uncompensated quartz crystal oscillatores and nominal update 1372 * intervals less than 1024 s, operation should be in phase-lock mode 1373 * (STA_FLL = 0), where the loop is disciplined to phase. For update 1374 * intervals greater than thiss, operation should be in frequency-lock 1375 * mode (STA_FLL = 1), where the loop is disciplined to frequency. 1376 * 1377 * Note: splclock() is in effect. 1378 */ 1379 void 1380 hardupdate(long offset) 1381 { 1382 long ltemp, mtemp; 1383 1384 if (!(time_status & STA_PLL) && !(time_status & STA_PPSTIME)) 1385 return; 1386 ltemp = offset; 1387 #ifdef PPS_SYNC 1388 if (time_status & STA_PPSTIME && time_status & STA_PPSSIGNAL) 1389 ltemp = pps_offset; 1390 #endif /* PPS_SYNC */ 1391 1392 /* 1393 * Scale the phase adjustment and clamp to the operating range. 1394 */ 1395 if (ltemp > MAXPHASE) 1396 time_offset = MAXPHASE << SHIFT_UPDATE; 1397 else if (ltemp < -MAXPHASE) 1398 time_offset = -(MAXPHASE << SHIFT_UPDATE); 1399 else 1400 time_offset = ltemp << SHIFT_UPDATE; 1401 1402 /* 1403 * Select whether the frequency is to be controlled and in which 1404 * mode (PLL or FLL). Clamp to the operating range. Ugly 1405 * multiply/divide should be replaced someday. 1406 */ 1407 if (time_status & STA_FREQHOLD || time_reftime == 0) 1408 time_reftime = time.tv_sec; 1409 mtemp = time.tv_sec - time_reftime; 1410 time_reftime = time.tv_sec; 1411 if (time_status & STA_FLL) { 1412 if (mtemp >= MINSEC) { 1413 ltemp = ((time_offset / mtemp) << (SHIFT_USEC - 1414 SHIFT_UPDATE)); 1415 if (ltemp < 0) 1416 time_freq -= -ltemp >> SHIFT_KH; 1417 else 1418 time_freq += ltemp >> SHIFT_KH; 1419 } 1420 } else { 1421 if (mtemp < MAXSEC) { 1422 ltemp *= mtemp; 1423 if (ltemp < 0) 1424 time_freq -= -ltemp >> (time_constant + 1425 time_constant + SHIFT_KF - 1426 SHIFT_USEC); 1427 else 1428 time_freq += ltemp >> (time_constant + 1429 time_constant + SHIFT_KF - 1430 SHIFT_USEC); 1431 } 1432 } 1433 if (time_freq > time_tolerance) 1434 time_freq = time_tolerance; 1435 else if (time_freq < -time_tolerance) 1436 time_freq = -time_tolerance; 1437 } 1438 1439 #ifdef PPS_SYNC 1440 /* 1441 * hardpps() - discipline CPU clock oscillator to external PPS signal 1442 * 1443 * This routine is called at each PPS interrupt in order to discipline 1444 * the CPU clock oscillator to the PPS signal. It measures the PPS phase 1445 * and leaves it in a handy spot for the hardclock() routine. It 1446 * integrates successive PPS phase differences and calculates the 1447 * frequency offset. This is used in hardclock() to discipline the CPU 1448 * clock oscillator so that intrinsic frequency error is cancelled out. 1449 * The code requires the caller to capture the time and hardware counter 1450 * value at the on-time PPS signal transition. 1451 * 1452 * Note that, on some Unix systems, this routine runs at an interrupt 1453 * priority level higher than the timer interrupt routine hardclock(). 1454 * Therefore, the variables used are distinct from the hardclock() 1455 * variables, except for certain exceptions: The PPS frequency pps_freq 1456 * and phase pps_offset variables are determined by this routine and 1457 * updated atomically. The time_tolerance variable can be considered a 1458 * constant, since it is infrequently changed, and then only when the 1459 * PPS signal is disabled. The watchdog counter pps_valid is updated 1460 * once per second by hardclock() and is atomically cleared in this 1461 * routine. 1462 */ 1463 void 1464 hardpps(struct timeval *tvp, /* time at PPS */ 1465 long usec /* hardware counter at PPS */) 1466 { 1467 long u_usec, v_usec, bigtick; 1468 long cal_sec, cal_usec; 1469 1470 /* 1471 * An occasional glitch can be produced when the PPS interrupt 1472 * occurs in the hardclock() routine before the time variable is 1473 * updated. Here the offset is discarded when the difference 1474 * between it and the last one is greater than tick/2, but not 1475 * if the interval since the first discard exceeds 30 s. 1476 */ 1477 time_status |= STA_PPSSIGNAL; 1478 time_status &= ~(STA_PPSJITTER | STA_PPSWANDER | STA_PPSERROR); 1479 pps_valid = 0; 1480 u_usec = -tvp->tv_usec; 1481 if (u_usec < -500000) 1482 u_usec += 1000000; 1483 v_usec = pps_offset - u_usec; 1484 if (v_usec < 0) 1485 v_usec = -v_usec; 1486 if (v_usec > (tick >> 1)) { 1487 if (pps_glitch > MAXGLITCH) { 1488 pps_glitch = 0; 1489 pps_tf[2] = u_usec; 1490 pps_tf[1] = u_usec; 1491 } else { 1492 pps_glitch++; 1493 u_usec = pps_offset; 1494 } 1495 } else 1496 pps_glitch = 0; 1497 1498 /* 1499 * A three-stage median filter is used to help deglitch the pps 1500 * time. The median sample becomes the time offset estimate; the 1501 * difference between the other two samples becomes the time 1502 * dispersion (jitter) estimate. 1503 */ 1504 pps_tf[2] = pps_tf[1]; 1505 pps_tf[1] = pps_tf[0]; 1506 pps_tf[0] = u_usec; 1507 if (pps_tf[0] > pps_tf[1]) { 1508 if (pps_tf[1] > pps_tf[2]) { 1509 pps_offset = pps_tf[1]; /* 0 1 2 */ 1510 v_usec = pps_tf[0] - pps_tf[2]; 1511 } else if (pps_tf[2] > pps_tf[0]) { 1512 pps_offset = pps_tf[0]; /* 2 0 1 */ 1513 v_usec = pps_tf[2] - pps_tf[1]; 1514 } else { 1515 pps_offset = pps_tf[2]; /* 0 2 1 */ 1516 v_usec = pps_tf[0] - pps_tf[1]; 1517 } 1518 } else { 1519 if (pps_tf[1] < pps_tf[2]) { 1520 pps_offset = pps_tf[1]; /* 2 1 0 */ 1521 v_usec = pps_tf[2] - pps_tf[0]; 1522 } else if (pps_tf[2] < pps_tf[0]) { 1523 pps_offset = pps_tf[0]; /* 1 0 2 */ 1524 v_usec = pps_tf[1] - pps_tf[2]; 1525 } else { 1526 pps_offset = pps_tf[2]; /* 1 2 0 */ 1527 v_usec = pps_tf[1] - pps_tf[0]; 1528 } 1529 } 1530 if (v_usec > MAXTIME) 1531 pps_jitcnt++; 1532 v_usec = (v_usec << PPS_AVG) - pps_jitter; 1533 if (v_usec < 0) 1534 pps_jitter -= -v_usec >> PPS_AVG; 1535 else 1536 pps_jitter += v_usec >> PPS_AVG; 1537 if (pps_jitter > (MAXTIME >> 1)) 1538 time_status |= STA_PPSJITTER; 1539 1540 /* 1541 * During the calibration interval adjust the starting time when 1542 * the tick overflows. At the end of the interval compute the 1543 * duration of the interval and the difference of the hardware 1544 * counters at the beginning and end of the interval. This code 1545 * is deliciously complicated by the fact valid differences may 1546 * exceed the value of tick when using long calibration 1547 * intervals and small ticks. Note that the counter can be 1548 * greater than tick if caught at just the wrong instant, but 1549 * the values returned and used here are correct. 1550 */ 1551 bigtick = (long)tick << SHIFT_USEC; 1552 pps_usec -= pps_freq; 1553 if (pps_usec >= bigtick) 1554 pps_usec -= bigtick; 1555 if (pps_usec < 0) 1556 pps_usec += bigtick; 1557 pps_time.tv_sec++; 1558 pps_count++; 1559 if (pps_count < (1 << pps_shift)) 1560 return; 1561 pps_count = 0; 1562 pps_calcnt++; 1563 u_usec = usec << SHIFT_USEC; 1564 v_usec = pps_usec - u_usec; 1565 if (v_usec >= bigtick >> 1) 1566 v_usec -= bigtick; 1567 if (v_usec < -(bigtick >> 1)) 1568 v_usec += bigtick; 1569 if (v_usec < 0) 1570 v_usec = -(-v_usec >> pps_shift); 1571 else 1572 v_usec = v_usec >> pps_shift; 1573 pps_usec = u_usec; 1574 cal_sec = tvp->tv_sec; 1575 cal_usec = tvp->tv_usec; 1576 cal_sec -= pps_time.tv_sec; 1577 cal_usec -= pps_time.tv_usec; 1578 if (cal_usec < 0) { 1579 cal_usec += 1000000; 1580 cal_sec--; 1581 } 1582 pps_time = *tvp; 1583 1584 /* 1585 * Check for lost interrupts, noise, excessive jitter and 1586 * excessive frequency error. The number of timer ticks during 1587 * the interval may vary +-1 tick. Add to this a margin of one 1588 * tick for the PPS signal jitter and maximum frequency 1589 * deviation. If the limits are exceeded, the calibration 1590 * interval is reset to the minimum and we start over. 1591 */ 1592 u_usec = (long)tick << 1; 1593 if (!((cal_sec == -1 && cal_usec > (1000000 - u_usec)) 1594 || (cal_sec == 0 && cal_usec < u_usec)) 1595 || v_usec > time_tolerance || v_usec < -time_tolerance) { 1596 pps_errcnt++; 1597 pps_shift = PPS_SHIFT; 1598 pps_intcnt = 0; 1599 time_status |= STA_PPSERROR; 1600 return; 1601 } 1602 1603 /* 1604 * A three-stage median filter is used to help deglitch the pps 1605 * frequency. The median sample becomes the frequency offset 1606 * estimate; the difference between the other two samples 1607 * becomes the frequency dispersion (stability) estimate. 1608 */ 1609 pps_ff[2] = pps_ff[1]; 1610 pps_ff[1] = pps_ff[0]; 1611 pps_ff[0] = v_usec; 1612 if (pps_ff[0] > pps_ff[1]) { 1613 if (pps_ff[1] > pps_ff[2]) { 1614 u_usec = pps_ff[1]; /* 0 1 2 */ 1615 v_usec = pps_ff[0] - pps_ff[2]; 1616 } else if (pps_ff[2] > pps_ff[0]) { 1617 u_usec = pps_ff[0]; /* 2 0 1 */ 1618 v_usec = pps_ff[2] - pps_ff[1]; 1619 } else { 1620 u_usec = pps_ff[2]; /* 0 2 1 */ 1621 v_usec = pps_ff[0] - pps_ff[1]; 1622 } 1623 } else { 1624 if (pps_ff[1] < pps_ff[2]) { 1625 u_usec = pps_ff[1]; /* 2 1 0 */ 1626 v_usec = pps_ff[2] - pps_ff[0]; 1627 } else if (pps_ff[2] < pps_ff[0]) { 1628 u_usec = pps_ff[0]; /* 1 0 2 */ 1629 v_usec = pps_ff[1] - pps_ff[2]; 1630 } else { 1631 u_usec = pps_ff[2]; /* 1 2 0 */ 1632 v_usec = pps_ff[1] - pps_ff[0]; 1633 } 1634 } 1635 1636 /* 1637 * Here the frequency dispersion (stability) is updated. If it 1638 * is less than one-fourth the maximum (MAXFREQ), the frequency 1639 * offset is updated as well, but clamped to the tolerance. It 1640 * will be processed later by the hardclock() routine. 1641 */ 1642 v_usec = (v_usec >> 1) - pps_stabil; 1643 if (v_usec < 0) 1644 pps_stabil -= -v_usec >> PPS_AVG; 1645 else 1646 pps_stabil += v_usec >> PPS_AVG; 1647 if (pps_stabil > MAXFREQ >> 2) { 1648 pps_stbcnt++; 1649 time_status |= STA_PPSWANDER; 1650 return; 1651 } 1652 if (time_status & STA_PPSFREQ) { 1653 if (u_usec < 0) { 1654 pps_freq -= -u_usec >> PPS_AVG; 1655 if (pps_freq < -time_tolerance) 1656 pps_freq = -time_tolerance; 1657 u_usec = -u_usec; 1658 } else { 1659 pps_freq += u_usec >> PPS_AVG; 1660 if (pps_freq > time_tolerance) 1661 pps_freq = time_tolerance; 1662 } 1663 } 1664 1665 /* 1666 * Here the calibration interval is adjusted. If the maximum 1667 * time difference is greater than tick / 4, reduce the interval 1668 * by half. If this is not the case for four consecutive 1669 * intervals, double the interval. 1670 */ 1671 if (u_usec << pps_shift > bigtick >> 2) { 1672 pps_intcnt = 0; 1673 if (pps_shift > PPS_SHIFT) 1674 pps_shift--; 1675 } else if (pps_intcnt >= 4) { 1676 pps_intcnt = 0; 1677 if (pps_shift < PPS_SHIFTMAX) 1678 pps_shift++; 1679 } else 1680 pps_intcnt++; 1681 } 1682 #endif /* PPS_SYNC */ 1683 #endif /* NTP */ 1684 1685 /* 1686 * Return information about system clocks. 1687 */ 1688 int 1689 sysctl_clockrate(void *where, size_t *sizep) 1690 { 1691 struct clockinfo clkinfo; 1692 1693 /* 1694 * Construct clockinfo structure. 1695 */ 1696 clkinfo.tick = tick; 1697 clkinfo.tickadj = tickadj; 1698 clkinfo.hz = hz; 1699 clkinfo.profhz = profhz; 1700 clkinfo.stathz = stathz ? stathz : hz; 1701 return (sysctl_rdstruct(where, sizep, NULL, &clkinfo, sizeof(clkinfo))); 1702 } 1703