xref: /dflybsd-src/sys/kern/kern_clock.c (revision c5f0c4c21ba588273de660f6d30818ff3858441c)
1 /*
2  * Copyright (c) 2003,2004 The DragonFly Project.  All rights reserved.
3  *
4  * This code is derived from software contributed to The DragonFly Project
5  * by Matthew Dillon <dillon@backplane.com>
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in
15  *    the documentation and/or other materials provided with the
16  *    distribution.
17  * 3. Neither the name of The DragonFly Project nor the names of its
18  *    contributors may be used to endorse or promote products derived
19  *    from this software without specific, prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  * Copyright (c) 1997, 1998 Poul-Henning Kamp <phk@FreeBSD.org>
35  * Copyright (c) 1982, 1986, 1991, 1993
36  *	The Regents of the University of California.  All rights reserved.
37  * (c) UNIX System Laboratories, Inc.
38  * All or some portions of this file are derived from material licensed
39  * to the University of California by American Telephone and Telegraph
40  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
41  * the permission of UNIX System Laboratories, Inc.
42  *
43  * Redistribution and use in source and binary forms, with or without
44  * modification, are permitted provided that the following conditions
45  * are met:
46  * 1. Redistributions of source code must retain the above copyright
47  *    notice, this list of conditions and the following disclaimer.
48  * 2. Redistributions in binary form must reproduce the above copyright
49  *    notice, this list of conditions and the following disclaimer in the
50  *    documentation and/or other materials provided with the distribution.
51  * 3. Neither the name of the University nor the names of its contributors
52  *    may be used to endorse or promote products derived from this software
53  *    without specific prior written permission.
54  *
55  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
56  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
57  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
58  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
59  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
60  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
61  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
62  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
63  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
64  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
65  * SUCH DAMAGE.
66  *
67  *	@(#)kern_clock.c	8.5 (Berkeley) 1/21/94
68  * $FreeBSD: src/sys/kern/kern_clock.c,v 1.105.2.10 2002/10/17 13:19:40 maxim Exp $
69  */
70 
71 #include "opt_ntp.h"
72 #include "opt_ifpoll.h"
73 #include "opt_pctrack.h"
74 
75 #include <sys/param.h>
76 #include <sys/systm.h>
77 #include <sys/callout.h>
78 #include <sys/kernel.h>
79 #include <sys/kinfo.h>
80 #include <sys/proc.h>
81 #include <sys/malloc.h>
82 #include <sys/resource.h>
83 #include <sys/resourcevar.h>
84 #include <sys/signalvar.h>
85 #include <sys/priv.h>
86 #include <sys/timex.h>
87 #include <sys/timepps.h>
88 #include <sys/upmap.h>
89 #include <vm/vm.h>
90 #include <sys/lock.h>
91 #include <vm/pmap.h>
92 #include <vm/vm_map.h>
93 #include <vm/vm_extern.h>
94 #include <sys/sysctl.h>
95 
96 #include <sys/thread2.h>
97 #include <sys/spinlock2.h>
98 
99 #include <machine/cpu.h>
100 #include <machine/limits.h>
101 #include <machine/smp.h>
102 #include <machine/cpufunc.h>
103 #include <machine/specialreg.h>
104 #include <machine/clock.h>
105 
106 #ifdef GPROF
107 #include <sys/gmon.h>
108 #endif
109 
110 #ifdef IFPOLL_ENABLE
111 extern void ifpoll_init_pcpu(int);
112 #endif
113 
114 #ifdef DEBUG_PCTRACK
115 static void do_pctrack(struct intrframe *frame, int which);
116 #endif
117 
118 static void initclocks (void *dummy);
119 SYSINIT(clocks, SI_BOOT2_CLOCKS, SI_ORDER_FIRST, initclocks, NULL);
120 
121 /*
122  * Some of these don't belong here, but it's easiest to concentrate them.
123  * Note that cpu_time counts in microseconds, but most userland programs
124  * just compare relative times against the total by delta.
125  */
126 struct kinfo_cputime cputime_percpu[MAXCPU];
127 #ifdef DEBUG_PCTRACK
128 struct kinfo_pcheader cputime_pcheader = { PCTRACK_SIZE, PCTRACK_ARYSIZE };
129 struct kinfo_pctrack cputime_pctrack[MAXCPU][PCTRACK_SIZE];
130 #endif
131 
132 static int
133 sysctl_cputime(SYSCTL_HANDLER_ARGS)
134 {
135 	int cpu, error = 0;
136 	int root_error;
137 	size_t size = sizeof(struct kinfo_cputime);
138 	struct kinfo_cputime tmp;
139 
140 	/*
141 	 * NOTE: For security reasons, only root can sniff %rip
142 	 */
143 	root_error = priv_check_cred(curthread->td_ucred, PRIV_ROOT, 0);
144 
145 	for (cpu = 0; cpu < ncpus; ++cpu) {
146 		tmp = cputime_percpu[cpu];
147 		if (root_error == 0) {
148 			tmp.cp_sample_pc =
149 				(int64_t)globaldata_find(cpu)->gd_sample_pc;
150 			tmp.cp_sample_sp =
151 				(int64_t)globaldata_find(cpu)->gd_sample_sp;
152 		}
153 		if ((error = SYSCTL_OUT(req, &tmp, size)) != 0)
154 			break;
155 	}
156 
157 	if (root_error == 0)
158 		smp_sniff();
159 
160 	return (error);
161 }
162 SYSCTL_PROC(_kern, OID_AUTO, cputime, (CTLTYPE_OPAQUE|CTLFLAG_RD), 0, 0,
163 	sysctl_cputime, "S,kinfo_cputime", "CPU time statistics");
164 
165 static int
166 sysctl_cp_time(SYSCTL_HANDLER_ARGS)
167 {
168 	long cpu_states[5] = {0};
169 	int cpu, error = 0;
170 	size_t size = sizeof(cpu_states);
171 
172 	for (cpu = 0; cpu < ncpus; ++cpu) {
173 		cpu_states[CP_USER] += cputime_percpu[cpu].cp_user;
174 		cpu_states[CP_NICE] += cputime_percpu[cpu].cp_nice;
175 		cpu_states[CP_SYS] += cputime_percpu[cpu].cp_sys;
176 		cpu_states[CP_INTR] += cputime_percpu[cpu].cp_intr;
177 		cpu_states[CP_IDLE] += cputime_percpu[cpu].cp_idle;
178 	}
179 
180 	error = SYSCTL_OUT(req, cpu_states, size);
181 
182 	return (error);
183 }
184 
185 SYSCTL_PROC(_kern, OID_AUTO, cp_time, (CTLTYPE_LONG|CTLFLAG_RD), 0, 0,
186 	sysctl_cp_time, "LU", "CPU time statistics");
187 
188 /*
189  * boottime is used to calculate the 'real' uptime.  Do not confuse this with
190  * microuptime().  microtime() is not drift compensated.  The real uptime
191  * with compensation is nanotime() - bootime.  boottime is recalculated
192  * whenever the real time is set based on the compensated elapsed time
193  * in seconds (gd->gd_time_seconds).
194  *
195  * The gd_time_seconds and gd_cpuclock_base fields remain fairly monotonic.
196  * Slight adjustments to gd_cpuclock_base are made to phase-lock it to
197  * the real time.
198  *
199  * WARNING! time_second can backstep on time corrections. Also, unlike
200  *          time_second, time_uptime is not a "real" time_t (seconds
201  *          since the Epoch) but seconds since booting.
202  */
203 struct timespec boottime;	/* boot time (realtime) for reference only */
204 time_t time_second;		/* read-only 'passive' realtime in seconds */
205 time_t time_uptime;		/* read-only 'passive' uptime in seconds */
206 
207 /*
208  * basetime is used to calculate the compensated real time of day.  The
209  * basetime can be modified on a per-tick basis by the adjtime(),
210  * ntp_adjtime(), and sysctl-based time correction APIs.
211  *
212  * Note that frequency corrections can also be made by adjusting
213  * gd_cpuclock_base.
214  *
215  * basetime is a tail-chasing FIFO, updated only by cpu #0.  The FIFO is
216  * used on both SMP and UP systems to avoid MP races between cpu's and
217  * interrupt races on UP systems.
218  */
219 struct hardtime {
220 	__uint32_t time_second;
221 	sysclock_t cpuclock_base;
222 };
223 
224 #define BASETIME_ARYSIZE	16
225 #define BASETIME_ARYMASK	(BASETIME_ARYSIZE - 1)
226 static struct timespec basetime[BASETIME_ARYSIZE];
227 static struct hardtime hardtime[BASETIME_ARYSIZE];
228 static volatile int basetime_index;
229 
230 static int
231 sysctl_get_basetime(SYSCTL_HANDLER_ARGS)
232 {
233 	struct timespec *bt;
234 	int error;
235 	int index;
236 
237 	/*
238 	 * Because basetime data and index may be updated by another cpu,
239 	 * a load fence is required to ensure that the data we read has
240 	 * not been speculatively read relative to a possibly updated index.
241 	 */
242 	index = basetime_index;
243 	cpu_lfence();
244 	bt = &basetime[index];
245 	error = SYSCTL_OUT(req, bt, sizeof(*bt));
246 	return (error);
247 }
248 
249 SYSCTL_STRUCT(_kern, KERN_BOOTTIME, boottime, CTLFLAG_RD,
250     &boottime, timespec, "System boottime");
251 SYSCTL_PROC(_kern, OID_AUTO, basetime, CTLTYPE_STRUCT|CTLFLAG_RD, 0, 0,
252     sysctl_get_basetime, "S,timespec", "System basetime");
253 
254 static void hardclock(systimer_t info, int, struct intrframe *frame);
255 static void statclock(systimer_t info, int, struct intrframe *frame);
256 static void schedclock(systimer_t info, int, struct intrframe *frame);
257 static void getnanotime_nbt(struct timespec *nbt, struct timespec *tsp);
258 
259 int	ticks;			/* system master ticks at hz */
260 int	clocks_running;		/* tsleep/timeout clocks operational */
261 int64_t	nsec_adj;		/* ntpd per-tick adjustment in nsec << 32 */
262 int64_t	nsec_acc;		/* accumulator */
263 int	sched_ticks;		/* global schedule clock ticks */
264 
265 /* NTPD time correction fields */
266 int64_t	ntp_tick_permanent;	/* per-tick adjustment in nsec << 32 */
267 int64_t	ntp_tick_acc;		/* accumulator for per-tick adjustment */
268 int64_t	ntp_delta;		/* one-time correction in nsec */
269 int64_t ntp_big_delta = 1000000000;
270 int32_t	ntp_tick_delta;		/* current adjustment rate */
271 int32_t	ntp_default_tick_delta;	/* adjustment rate for ntp_delta */
272 time_t	ntp_leap_second;	/* time of next leap second */
273 int	ntp_leap_insert;	/* whether to insert or remove a second */
274 struct spinlock ntp_spin;
275 
276 /*
277  * Finish initializing clock frequencies and start all clocks running.
278  */
279 /* ARGSUSED*/
280 static void
281 initclocks(void *dummy)
282 {
283 	/*psratio = profhz / stathz;*/
284 	spin_init(&ntp_spin, "ntp");
285 	initclocks_pcpu();
286 	clocks_running = 1;
287 	if (kpmap) {
288 	    kpmap->tsc_freq = (uint64_t)tsc_frequency;
289 	    kpmap->tick_freq = hz;
290 	}
291 }
292 
293 /*
294  * Called on a per-cpu basis from the idle thread bootstrap on each cpu
295  * during SMP initialization.
296  *
297  * This routine is called concurrently during low-level SMP initialization
298  * and may not block in any way.  Meaning, among other things, we can't
299  * acquire any tokens.
300  */
301 void
302 initclocks_pcpu(void)
303 {
304 	struct globaldata *gd = mycpu;
305 
306 	crit_enter();
307 	if (gd->gd_cpuid == 0) {
308 	    gd->gd_time_seconds = 1;
309 	    gd->gd_cpuclock_base = sys_cputimer->count();
310 	    hardtime[0].time_second = gd->gd_time_seconds;
311 	    hardtime[0].cpuclock_base = gd->gd_cpuclock_base;
312 	} else {
313 	    gd->gd_time_seconds = globaldata_find(0)->gd_time_seconds;
314 	    gd->gd_cpuclock_base = globaldata_find(0)->gd_cpuclock_base;
315 	}
316 
317 	systimer_intr_enable();
318 
319 	crit_exit();
320 }
321 
322 /*
323  * This routine is called on just the BSP, just after SMP initialization
324  * completes to * finish initializing any clocks that might contend/block
325  * (e.g. like on a token).  We can't do this in initclocks_pcpu() because
326  * that function is called from the idle thread bootstrap for each cpu and
327  * not allowed to block at all.
328  */
329 static
330 void
331 initclocks_other(void *dummy)
332 {
333 	struct globaldata *ogd = mycpu;
334 	struct globaldata *gd;
335 	int n;
336 
337 	for (n = 0; n < ncpus; ++n) {
338 		lwkt_setcpu_self(globaldata_find(n));
339 		gd = mycpu;
340 
341 		/*
342 		 * Use a non-queued periodic systimer to prevent multiple
343 		 * ticks from building up if the sysclock jumps forward
344 		 * (8254 gets reset).  The sysclock will never jump backwards.
345 		 * Our time sync is based on the actual sysclock, not the
346 		 * ticks count.
347 		 */
348 		systimer_init_periodic_nq(&gd->gd_hardclock, hardclock,
349 					  NULL, hz);
350 		systimer_init_periodic_nq(&gd->gd_statclock, statclock,
351 					  NULL, stathz);
352 		/* XXX correct the frequency for scheduler / estcpu tests */
353 		systimer_init_periodic_nq(&gd->gd_schedclock, schedclock,
354 					  NULL, ESTCPUFREQ);
355 #ifdef IFPOLL_ENABLE
356 		ifpoll_init_pcpu(gd->gd_cpuid);
357 #endif
358 	}
359 	lwkt_setcpu_self(ogd);
360 }
361 SYSINIT(clocks2, SI_BOOT2_POST_SMP, SI_ORDER_ANY, initclocks_other, NULL);
362 
363 /*
364  * This sets the current real time of day.  Timespecs are in seconds and
365  * nanoseconds.  We do not mess with gd_time_seconds and gd_cpuclock_base,
366  * instead we adjust basetime so basetime + gd_* results in the current
367  * time of day.  This way the gd_* fields are guaranteed to represent
368  * a monotonically increasing 'uptime' value.
369  *
370  * When set_timeofday() is called from userland, the system call forces it
371  * onto cpu #0 since only cpu #0 can update basetime_index.
372  */
373 void
374 set_timeofday(struct timespec *ts)
375 {
376 	struct timespec *nbt;
377 	int ni;
378 
379 	/*
380 	 * XXX SMP / non-atomic basetime updates
381 	 */
382 	crit_enter();
383 	ni = (basetime_index + 1) & BASETIME_ARYMASK;
384 	cpu_lfence();
385 	nbt = &basetime[ni];
386 	nanouptime(nbt);
387 	nbt->tv_sec = ts->tv_sec - nbt->tv_sec;
388 	nbt->tv_nsec = ts->tv_nsec - nbt->tv_nsec;
389 	if (nbt->tv_nsec < 0) {
390 	    nbt->tv_nsec += 1000000000;
391 	    --nbt->tv_sec;
392 	}
393 
394 	/*
395 	 * Note that basetime diverges from boottime as the clock drift is
396 	 * compensated for, so we cannot do away with boottime.  When setting
397 	 * the absolute time of day the drift is 0 (for an instant) and we
398 	 * can simply assign boottime to basetime.
399 	 *
400 	 * Note that nanouptime() is based on gd_time_seconds which is drift
401 	 * compensated up to a point (it is guaranteed to remain monotonically
402 	 * increasing).  gd_time_seconds is thus our best uptime guess and
403 	 * suitable for use in the boottime calculation.  It is already taken
404 	 * into account in the basetime calculation above.
405 	 */
406 	spin_lock(&ntp_spin);
407 	boottime.tv_sec = nbt->tv_sec;
408 	ntp_delta = 0;
409 
410 	/*
411 	 * We now have a new basetime, make sure all other cpus have it,
412 	 * then update the index.
413 	 */
414 	cpu_sfence();
415 	basetime_index = ni;
416 	spin_unlock(&ntp_spin);
417 
418 	crit_exit();
419 }
420 
421 /*
422  * Each cpu has its own hardclock, but we only increments ticks and softticks
423  * on cpu #0.
424  *
425  * NOTE! systimer! the MP lock might not be held here.  We can only safely
426  * manipulate objects owned by the current cpu.
427  */
428 static void
429 hardclock(systimer_t info, int in_ipi, struct intrframe *frame)
430 {
431 	sysclock_t cputicks;
432 	struct proc *p;
433 	struct globaldata *gd = mycpu;
434 
435 	if ((gd->gd_reqflags & RQF_IPIQ) == 0 && lwkt_need_ipiq_process(gd)) {
436 		/* Defer to doreti on passive IPIQ processing */
437 		need_ipiq();
438 	}
439 
440 	/*
441 	 * We update the compensation base to calculate fine-grained time
442 	 * from the sys_cputimer on a per-cpu basis in order to avoid
443 	 * having to mess around with locks.  sys_cputimer is assumed to
444 	 * be consistent across all cpus.  CPU N copies the base state from
445 	 * CPU 0 using the same FIFO trick that we use for basetime (so we
446 	 * don't catch a CPU 0 update in the middle).
447 	 *
448 	 * Note that we never allow info->time (aka gd->gd_hardclock.time)
449 	 * to reverse index gd_cpuclock_base, but that it is possible for
450 	 * it to temporarily get behind in the seconds if something in the
451 	 * system locks interrupts for a long period of time.  Since periodic
452 	 * timers count events, though everything should resynch again
453 	 * immediately.
454 	 */
455 	if (gd->gd_cpuid == 0) {
456 		int ni;
457 
458 		cputicks = info->time - gd->gd_cpuclock_base;
459 		if (cputicks >= sys_cputimer->freq) {
460 			cputicks /= sys_cputimer->freq;
461 			if (cputicks != 0 && cputicks != 1)
462 				kprintf("Warning: hardclock missed > 1 sec\n");
463 			gd->gd_time_seconds += cputicks;
464 			gd->gd_cpuclock_base += sys_cputimer->freq * cputicks;
465 			/* uncorrected monotonic 1-sec gran */
466 			time_uptime += cputicks;
467 		}
468 		ni = (basetime_index + 1) & BASETIME_ARYMASK;
469 		hardtime[ni].time_second = gd->gd_time_seconds;
470 		hardtime[ni].cpuclock_base = gd->gd_cpuclock_base;
471 	} else {
472 		int ni;
473 
474 		ni = basetime_index;
475 		cpu_lfence();
476 		gd->gd_time_seconds = hardtime[ni].time_second;
477 		gd->gd_cpuclock_base = hardtime[ni].cpuclock_base;
478 	}
479 
480 	/*
481 	 * The system-wide ticks counter and NTP related timedelta/tickdelta
482 	 * adjustments only occur on cpu #0.  NTP adjustments are accomplished
483 	 * by updating basetime.
484 	 */
485 	if (gd->gd_cpuid == 0) {
486 	    struct timespec *nbt;
487 	    struct timespec nts;
488 	    int leap;
489 	    int ni;
490 
491 	    ++ticks;
492 
493 #if 0
494 	    if (tco->tc_poll_pps)
495 		tco->tc_poll_pps(tco);
496 #endif
497 
498 	    /*
499 	     * Calculate the new basetime index.  We are in a critical section
500 	     * on cpu #0 and can safely play with basetime_index.  Start
501 	     * with the current basetime and then make adjustments.
502 	     */
503 	    ni = (basetime_index + 1) & BASETIME_ARYMASK;
504 	    nbt = &basetime[ni];
505 	    *nbt = basetime[basetime_index];
506 
507 	    /*
508 	     * ntp adjustments only occur on cpu 0 and are protected by
509 	     * ntp_spin.  This spinlock virtually never conflicts.
510 	     */
511 	    spin_lock(&ntp_spin);
512 
513 	    /*
514 	     * Apply adjtime corrections.  (adjtime() API)
515 	     *
516 	     * adjtime() only runs on cpu #0 so our critical section is
517 	     * sufficient to access these variables.
518 	     */
519 	    if (ntp_delta != 0) {
520 		nbt->tv_nsec += ntp_tick_delta;
521 		ntp_delta -= ntp_tick_delta;
522 		if ((ntp_delta > 0 && ntp_delta < ntp_tick_delta) ||
523 		    (ntp_delta < 0 && ntp_delta > ntp_tick_delta)) {
524 			ntp_tick_delta = ntp_delta;
525  		}
526  	    }
527 
528 	    /*
529 	     * Apply permanent frequency corrections.  (sysctl API)
530 	     */
531 	    if (ntp_tick_permanent != 0) {
532 		ntp_tick_acc += ntp_tick_permanent;
533 		if (ntp_tick_acc >= (1LL << 32)) {
534 		    nbt->tv_nsec += ntp_tick_acc >> 32;
535 		    ntp_tick_acc -= (ntp_tick_acc >> 32) << 32;
536 		} else if (ntp_tick_acc <= -(1LL << 32)) {
537 		    /* Negate ntp_tick_acc to avoid shifting the sign bit. */
538 		    nbt->tv_nsec -= (-ntp_tick_acc) >> 32;
539 		    ntp_tick_acc += ((-ntp_tick_acc) >> 32) << 32;
540 		}
541  	    }
542 
543 	    if (nbt->tv_nsec >= 1000000000) {
544 		    nbt->tv_sec++;
545 		    nbt->tv_nsec -= 1000000000;
546 	    } else if (nbt->tv_nsec < 0) {
547 		    nbt->tv_sec--;
548 		    nbt->tv_nsec += 1000000000;
549 	    }
550 
551 	    /*
552 	     * Another per-tick compensation.  (for ntp_adjtime() API)
553 	     */
554 	    if (nsec_adj != 0) {
555 		nsec_acc += nsec_adj;
556 		if (nsec_acc >= 0x100000000LL) {
557 		    nbt->tv_nsec += nsec_acc >> 32;
558 		    nsec_acc = (nsec_acc & 0xFFFFFFFFLL);
559 		} else if (nsec_acc <= -0x100000000LL) {
560 		    nbt->tv_nsec -= -nsec_acc >> 32;
561 		    nsec_acc = -(-nsec_acc & 0xFFFFFFFFLL);
562 		}
563 		if (nbt->tv_nsec >= 1000000000) {
564 		    nbt->tv_nsec -= 1000000000;
565 		    ++nbt->tv_sec;
566 		} else if (nbt->tv_nsec < 0) {
567 		    nbt->tv_nsec += 1000000000;
568 		    --nbt->tv_sec;
569 		}
570 	    }
571 	    spin_unlock(&ntp_spin);
572 
573 	    /************************************************************
574 	     *			LEAP SECOND CORRECTION			*
575 	     ************************************************************
576 	     *
577 	     * Taking into account all the corrections made above, figure
578 	     * out the new real time.  If the seconds field has changed
579 	     * then apply any pending leap-second corrections.
580 	     */
581 	    getnanotime_nbt(nbt, &nts);
582 
583 	    if (time_second != nts.tv_sec) {
584 		/*
585 		 * Apply leap second (sysctl API).  Adjust nts for changes
586 		 * so we do not have to call getnanotime_nbt again.
587 		 */
588 		if (ntp_leap_second) {
589 		    if (ntp_leap_second == nts.tv_sec) {
590 			if (ntp_leap_insert) {
591 			    nbt->tv_sec++;
592 			    nts.tv_sec++;
593 			} else {
594 			    nbt->tv_sec--;
595 			    nts.tv_sec--;
596 			}
597 			ntp_leap_second--;
598 		    }
599 		}
600 
601 		/*
602 		 * Apply leap second (ntp_adjtime() API), calculate a new
603 		 * nsec_adj field.  ntp_update_second() returns nsec_adj
604 		 * as a per-second value but we need it as a per-tick value.
605 		 */
606 		leap = ntp_update_second(time_second, &nsec_adj);
607 		nsec_adj /= hz;
608 		nbt->tv_sec += leap;
609 		nts.tv_sec += leap;
610 
611 		/*
612 		 * Update the time_second 'approximate time' global.
613 		 */
614 		time_second = nts.tv_sec;
615 	    }
616 
617 	    /*
618 	     * Finally, our new basetime is ready to go live!
619 	     */
620 	    cpu_sfence();
621 	    basetime_index = ni;
622 
623 	    /*
624 	     * Update kpmap on each tick.  TS updates are integrated with
625 	     * fences and upticks allowing userland to read the data
626 	     * deterministically.
627 	     */
628 	    if (kpmap) {
629 		int w;
630 
631 		w = (kpmap->upticks + 1) & 1;
632 		getnanouptime(&kpmap->ts_uptime[w]);
633 		getnanotime(&kpmap->ts_realtime[w]);
634 		cpu_sfence();
635 		++kpmap->upticks;
636 		cpu_sfence();
637 	    }
638 	}
639 
640 	/*
641 	 * lwkt thread scheduler fair queueing
642 	 */
643 	lwkt_schedulerclock(curthread);
644 
645 	/*
646 	 * softticks are handled for all cpus
647 	 */
648 	hardclock_softtick(gd);
649 
650 	/*
651 	 * Rollup accumulated vmstats, copy-back for critical path checks.
652 	 */
653 	vmstats_rollup_cpu(gd);
654 	mycpu->gd_vmstats = vmstats;
655 
656 	/*
657 	 * ITimer handling is per-tick, per-cpu.
658 	 *
659 	 * We must acquire the per-process token in order for ksignal()
660 	 * to be non-blocking.  For the moment this requires an AST fault,
661 	 * the ksignal() cannot be safely issued from this hard interrupt.
662 	 *
663 	 * XXX Even the trytoken here isn't right, and itimer operation in
664 	 *     a multi threaded environment is going to be weird at the
665 	 *     very least.
666 	 */
667 	if ((p = curproc) != NULL && lwkt_trytoken(&p->p_token)) {
668 		crit_enter_hard();
669 		if (p->p_upmap)
670 			++p->p_upmap->runticks;
671 
672 		if (frame && CLKF_USERMODE(frame) &&
673 		    timevalisset(&p->p_timer[ITIMER_VIRTUAL].it_value) &&
674 		    itimerdecr(&p->p_timer[ITIMER_VIRTUAL], ustick) == 0) {
675 			p->p_flags |= P_SIGVTALRM;
676 			need_user_resched();
677 		}
678 		if (timevalisset(&p->p_timer[ITIMER_PROF].it_value) &&
679 		    itimerdecr(&p->p_timer[ITIMER_PROF], ustick) == 0) {
680 			p->p_flags |= P_SIGPROF;
681 			need_user_resched();
682 		}
683 		crit_exit_hard();
684 		lwkt_reltoken(&p->p_token);
685 	}
686 	setdelayed();
687 }
688 
689 /*
690  * The statistics clock typically runs at a 125Hz rate, and is intended
691  * to be frequency offset from the hardclock (typ 100Hz).  It is per-cpu.
692  *
693  * NOTE! systimer! the MP lock might not be held here.  We can only safely
694  * manipulate objects owned by the current cpu.
695  *
696  * The stats clock is responsible for grabbing a profiling sample.
697  * Most of the statistics are only used by user-level statistics programs.
698  * The main exceptions are p->p_uticks, p->p_sticks, p->p_iticks, and
699  * p->p_estcpu.
700  *
701  * Like the other clocks, the stat clock is called from what is effectively
702  * a fast interrupt, so the context should be the thread/process that got
703  * interrupted.
704  */
705 static void
706 statclock(systimer_t info, int in_ipi, struct intrframe *frame)
707 {
708 #ifdef GPROF
709 	struct gmonparam *g;
710 	int i;
711 #endif
712 	thread_t td;
713 	struct proc *p;
714 	int bump;
715 	sysclock_t cv;
716 	sysclock_t scv;
717 
718 	/*
719 	 * How big was our timeslice relative to the last time?  Calculate
720 	 * in microseconds.
721 	 *
722 	 * NOTE: Use of microuptime() is typically MPSAFE, but usually not
723 	 *	 during early boot.  Just use the systimer count to be nice
724 	 *	 to e.g. qemu.  The systimer has a better chance of being
725 	 *	 MPSAFE at early boot.
726 	 */
727 	cv = sys_cputimer->count();
728 	scv = mycpu->statint.gd_statcv;
729 	if (scv == 0) {
730 		bump = 1;
731 	} else {
732 		bump = (sys_cputimer->freq64_usec * (cv - scv)) >> 32;
733 		if (bump < 0)
734 			bump = 0;
735 		if (bump > 1000000)
736 			bump = 1000000;
737 	}
738 	mycpu->statint.gd_statcv = cv;
739 
740 #if 0
741 	stv = &mycpu->gd_stattv;
742 	if (stv->tv_sec == 0) {
743 	    bump = 1;
744 	} else {
745 	    bump = tv.tv_usec - stv->tv_usec +
746 		(tv.tv_sec - stv->tv_sec) * 1000000;
747 	    if (bump < 0)
748 		bump = 0;
749 	    if (bump > 1000000)
750 		bump = 1000000;
751 	}
752 	*stv = tv;
753 #endif
754 
755 	td = curthread;
756 	p = td->td_proc;
757 
758 	if (frame && CLKF_USERMODE(frame)) {
759 		/*
760 		 * Came from userland, handle user time and deal with
761 		 * possible process.
762 		 */
763 		if (p && (p->p_flags & P_PROFIL))
764 			addupc_intr(p, CLKF_PC(frame), 1);
765 		td->td_uticks += bump;
766 
767 		/*
768 		 * Charge the time as appropriate
769 		 */
770 		if (p && p->p_nice > NZERO)
771 			cpu_time.cp_nice += bump;
772 		else
773 			cpu_time.cp_user += bump;
774 	} else {
775 		int intr_nest = mycpu->gd_intr_nesting_level;
776 
777 		if (in_ipi) {
778 			/*
779 			 * IPI processing code will bump gd_intr_nesting_level
780 			 * up by one, which breaks following CLKF_INTR testing,
781 			 * so we subtract it by one here.
782 			 */
783 			--intr_nest;
784 		}
785 #ifdef GPROF
786 		/*
787 		 * Kernel statistics are just like addupc_intr, only easier.
788 		 */
789 		g = &_gmonparam;
790 		if (g->state == GMON_PROF_ON && frame) {
791 			i = CLKF_PC(frame) - g->lowpc;
792 			if (i < g->textsize) {
793 				i /= HISTFRACTION * sizeof(*g->kcount);
794 				g->kcount[i]++;
795 			}
796 		}
797 #endif
798 
799 #define IS_INTR_RUNNING	((frame && CLKF_INTR(intr_nest)) || CLKF_INTR_TD(td))
800 
801 		/*
802 		 * Came from kernel mode, so we were:
803 		 * - handling an interrupt,
804 		 * - doing syscall or trap work on behalf of the current
805 		 *   user process, or
806 		 * - spinning in the idle loop.
807 		 * Whichever it is, charge the time as appropriate.
808 		 * Note that we charge interrupts to the current process,
809 		 * regardless of whether they are ``for'' that process,
810 		 * so that we know how much of its real time was spent
811 		 * in ``non-process'' (i.e., interrupt) work.
812 		 *
813 		 * XXX assume system if frame is NULL.  A NULL frame
814 		 * can occur if ipi processing is done from a crit_exit().
815 		 */
816 		if (IS_INTR_RUNNING)
817 			td->td_iticks += bump;
818 		else
819 			td->td_sticks += bump;
820 
821 		if (IS_INTR_RUNNING) {
822 			/*
823 			 * If we interrupted an interrupt thread, well,
824 			 * count it as interrupt time.
825 			 */
826 #ifdef DEBUG_PCTRACK
827 			if (frame)
828 				do_pctrack(frame, PCTRACK_INT);
829 #endif
830 			cpu_time.cp_intr += bump;
831 		} else {
832 			if (td == &mycpu->gd_idlethread) {
833 				/*
834 				 * Even if the current thread is the idle
835 				 * thread it could be due to token contention
836 				 * in the LWKT scheduler.  Count such as
837 				 * system time.
838 				 */
839 				if (mycpu->gd_reqflags & RQF_IDLECHECK_WK_MASK)
840 					cpu_time.cp_sys += bump;
841 				else
842 					cpu_time.cp_idle += bump;
843 			} else {
844 				/*
845 				 * System thread was running.
846 				 */
847 #ifdef DEBUG_PCTRACK
848 				if (frame)
849 					do_pctrack(frame, PCTRACK_SYS);
850 #endif
851 				cpu_time.cp_sys += bump;
852 			}
853 		}
854 
855 #undef IS_INTR_RUNNING
856 	}
857 }
858 
859 #ifdef DEBUG_PCTRACK
860 /*
861  * Sample the PC when in the kernel or in an interrupt.  User code can
862  * retrieve the information and generate a histogram or other output.
863  */
864 
865 static void
866 do_pctrack(struct intrframe *frame, int which)
867 {
868 	struct kinfo_pctrack *pctrack;
869 
870 	pctrack = &cputime_pctrack[mycpu->gd_cpuid][which];
871 	pctrack->pc_array[pctrack->pc_index & PCTRACK_ARYMASK] =
872 		(void *)CLKF_PC(frame);
873 	++pctrack->pc_index;
874 }
875 
876 static int
877 sysctl_pctrack(SYSCTL_HANDLER_ARGS)
878 {
879 	struct kinfo_pcheader head;
880 	int error;
881 	int cpu;
882 	int ntrack;
883 
884 	head.pc_ntrack = PCTRACK_SIZE;
885 	head.pc_arysize = PCTRACK_ARYSIZE;
886 
887 	if ((error = SYSCTL_OUT(req, &head, sizeof(head))) != 0)
888 		return (error);
889 
890 	for (cpu = 0; cpu < ncpus; ++cpu) {
891 		for (ntrack = 0; ntrack < PCTRACK_SIZE; ++ntrack) {
892 			error = SYSCTL_OUT(req, &cputime_pctrack[cpu][ntrack],
893 					   sizeof(struct kinfo_pctrack));
894 			if (error)
895 				break;
896 		}
897 		if (error)
898 			break;
899 	}
900 	return (error);
901 }
902 SYSCTL_PROC(_kern, OID_AUTO, pctrack, (CTLTYPE_OPAQUE|CTLFLAG_RD), 0, 0,
903 	sysctl_pctrack, "S,kinfo_pcheader", "CPU PC tracking");
904 
905 #endif
906 
907 /*
908  * The scheduler clock typically runs at a 50Hz rate.  NOTE! systimer,
909  * the MP lock might not be held.  We can safely manipulate parts of curproc
910  * but that's about it.
911  *
912  * Each cpu has its own scheduler clock.
913  */
914 static void
915 schedclock(systimer_t info, int in_ipi __unused, struct intrframe *frame)
916 {
917 	struct lwp *lp;
918 	struct rusage *ru;
919 	struct vmspace *vm;
920 	long rss;
921 
922 	if ((lp = lwkt_preempted_proc()) != NULL) {
923 		/*
924 		 * Account for cpu time used and hit the scheduler.  Note
925 		 * that this call MUST BE MP SAFE, and the BGL IS NOT HELD
926 		 * HERE.
927 		 */
928 		++lp->lwp_cpticks;
929 		usched_schedulerclock(lp, info->periodic, info->time);
930 	} else {
931 		usched_schedulerclock(NULL, info->periodic, info->time);
932 	}
933 	if ((lp = curthread->td_lwp) != NULL) {
934 		/*
935 		 * Update resource usage integrals and maximums.
936 		 */
937 		if ((ru = &lp->lwp_proc->p_ru) &&
938 		    (vm = lp->lwp_proc->p_vmspace) != NULL) {
939 			ru->ru_ixrss += pgtok(vm->vm_tsize);
940 			ru->ru_idrss += pgtok(vm->vm_dsize);
941 			ru->ru_isrss += pgtok(vm->vm_ssize);
942 			if (lwkt_trytoken(&vm->vm_map.token)) {
943 				rss = pgtok(vmspace_resident_count(vm));
944 				if (ru->ru_maxrss < rss)
945 					ru->ru_maxrss = rss;
946 				lwkt_reltoken(&vm->vm_map.token);
947 			}
948 		}
949 	}
950 	/* Increment the global sched_ticks */
951 	if (mycpu->gd_cpuid == 0)
952 		++sched_ticks;
953 }
954 
955 /*
956  * Compute number of ticks for the specified amount of time.  The
957  * return value is intended to be used in a clock interrupt timed
958  * operation and guaranteed to meet or exceed the requested time.
959  * If the representation overflows, return INT_MAX.  The minimum return
960  * value is 1 ticks and the function will average the calculation up.
961  * If any value greater then 0 microseconds is supplied, a value
962  * of at least 2 will be returned to ensure that a near-term clock
963  * interrupt does not cause the timeout to occur (degenerately) early.
964  *
965  * Note that limit checks must take into account microseconds, which is
966  * done simply by using the smaller signed long maximum instead of
967  * the unsigned long maximum.
968  *
969  * If ints have 32 bits, then the maximum value for any timeout in
970  * 10ms ticks is 248 days.
971  */
972 int
973 tvtohz_high(struct timeval *tv)
974 {
975 	int ticks;
976 	long sec, usec;
977 
978 	sec = tv->tv_sec;
979 	usec = tv->tv_usec;
980 	if (usec < 0) {
981 		sec--;
982 		usec += 1000000;
983 	}
984 	if (sec < 0) {
985 #ifdef DIAGNOSTIC
986 		if (usec > 0) {
987 			sec++;
988 			usec -= 1000000;
989 		}
990 		kprintf("tvtohz_high: negative time difference "
991 			"%ld sec %ld usec\n",
992 			sec, usec);
993 #endif
994 		ticks = 1;
995 	} else if (sec <= INT_MAX / hz) {
996 		ticks = (int)(sec * hz +
997 			    ((u_long)usec + (ustick - 1)) / ustick) + 1;
998 	} else {
999 		ticks = INT_MAX;
1000 	}
1001 	return (ticks);
1002 }
1003 
1004 int
1005 tstohz_high(struct timespec *ts)
1006 {
1007 	int ticks;
1008 	long sec, nsec;
1009 
1010 	sec = ts->tv_sec;
1011 	nsec = ts->tv_nsec;
1012 	if (nsec < 0) {
1013 		sec--;
1014 		nsec += 1000000000;
1015 	}
1016 	if (sec < 0) {
1017 #ifdef DIAGNOSTIC
1018 		if (nsec > 0) {
1019 			sec++;
1020 			nsec -= 1000000000;
1021 		}
1022 		kprintf("tstohz_high: negative time difference "
1023 			"%ld sec %ld nsec\n",
1024 			sec, nsec);
1025 #endif
1026 		ticks = 1;
1027 	} else if (sec <= INT_MAX / hz) {
1028 		ticks = (int)(sec * hz +
1029 			    ((u_long)nsec + (nstick - 1)) / nstick) + 1;
1030 	} else {
1031 		ticks = INT_MAX;
1032 	}
1033 	return (ticks);
1034 }
1035 
1036 
1037 /*
1038  * Compute number of ticks for the specified amount of time, erroring on
1039  * the side of it being too low to ensure that sleeping the returned number
1040  * of ticks will not result in a late return.
1041  *
1042  * The supplied timeval may not be negative and should be normalized.  A
1043  * return value of 0 is possible if the timeval converts to less then
1044  * 1 tick.
1045  *
1046  * If ints have 32 bits, then the maximum value for any timeout in
1047  * 10ms ticks is 248 days.
1048  */
1049 int
1050 tvtohz_low(struct timeval *tv)
1051 {
1052 	int ticks;
1053 	long sec;
1054 
1055 	sec = tv->tv_sec;
1056 	if (sec <= INT_MAX / hz)
1057 		ticks = (int)(sec * hz + (u_long)tv->tv_usec / ustick);
1058 	else
1059 		ticks = INT_MAX;
1060 	return (ticks);
1061 }
1062 
1063 int
1064 tstohz_low(struct timespec *ts)
1065 {
1066 	int ticks;
1067 	long sec;
1068 
1069 	sec = ts->tv_sec;
1070 	if (sec <= INT_MAX / hz)
1071 		ticks = (int)(sec * hz + (u_long)ts->tv_nsec / nstick);
1072 	else
1073 		ticks = INT_MAX;
1074 	return (ticks);
1075 }
1076 
1077 /*
1078  * Start profiling on a process.
1079  *
1080  * Caller must hold p->p_token();
1081  *
1082  * Kernel profiling passes proc0 which never exits and hence
1083  * keeps the profile clock running constantly.
1084  */
1085 void
1086 startprofclock(struct proc *p)
1087 {
1088 	if ((p->p_flags & P_PROFIL) == 0) {
1089 		p->p_flags |= P_PROFIL;
1090 #if 0	/* XXX */
1091 		if (++profprocs == 1 && stathz != 0) {
1092 			crit_enter();
1093 			psdiv = psratio;
1094 			setstatclockrate(profhz);
1095 			crit_exit();
1096 		}
1097 #endif
1098 	}
1099 }
1100 
1101 /*
1102  * Stop profiling on a process.
1103  *
1104  * caller must hold p->p_token
1105  */
1106 void
1107 stopprofclock(struct proc *p)
1108 {
1109 	if (p->p_flags & P_PROFIL) {
1110 		p->p_flags &= ~P_PROFIL;
1111 #if 0	/* XXX */
1112 		if (--profprocs == 0 && stathz != 0) {
1113 			crit_enter();
1114 			psdiv = 1;
1115 			setstatclockrate(stathz);
1116 			crit_exit();
1117 		}
1118 #endif
1119 	}
1120 }
1121 
1122 /*
1123  * Return information about system clocks.
1124  */
1125 static int
1126 sysctl_kern_clockrate(SYSCTL_HANDLER_ARGS)
1127 {
1128 	struct kinfo_clockinfo clkinfo;
1129 	/*
1130 	 * Construct clockinfo structure.
1131 	 */
1132 	clkinfo.ci_hz = hz;
1133 	clkinfo.ci_tick = ustick;
1134 	clkinfo.ci_tickadj = ntp_default_tick_delta / 1000;
1135 	clkinfo.ci_profhz = profhz;
1136 	clkinfo.ci_stathz = stathz ? stathz : hz;
1137 	return (sysctl_handle_opaque(oidp, &clkinfo, sizeof clkinfo, req));
1138 }
1139 
1140 SYSCTL_PROC(_kern, KERN_CLOCKRATE, clockrate, CTLTYPE_STRUCT|CTLFLAG_RD,
1141 	0, 0, sysctl_kern_clockrate, "S,clockinfo","");
1142 
1143 /*
1144  * We have eight functions for looking at the clock, four for
1145  * microseconds and four for nanoseconds.  For each there is fast
1146  * but less precise version "get{nano|micro}[up]time" which will
1147  * return a time which is up to 1/HZ previous to the call, whereas
1148  * the raw version "{nano|micro}[up]time" will return a timestamp
1149  * which is as precise as possible.  The "up" variants return the
1150  * time relative to system boot, these are well suited for time
1151  * interval measurements.
1152  *
1153  * Each cpu independently maintains the current time of day, so all
1154  * we need to do to protect ourselves from changes is to do a loop
1155  * check on the seconds field changing out from under us.
1156  *
1157  * The system timer maintains a 32 bit count and due to various issues
1158  * it is possible for the calculated delta to occasionally exceed
1159  * sys_cputimer->freq.  If this occurs the sys_cputimer->freq64_nsec
1160  * multiplication can easily overflow, so we deal with the case.  For
1161  * uniformity we deal with the case in the usec case too.
1162  *
1163  * All the [get][micro,nano][time,uptime]() routines are MPSAFE.
1164  */
1165 void
1166 getmicrouptime(struct timeval *tvp)
1167 {
1168 	struct globaldata *gd = mycpu;
1169 	sysclock_t delta;
1170 
1171 	do {
1172 		tvp->tv_sec = gd->gd_time_seconds;
1173 		delta = gd->gd_hardclock.time - gd->gd_cpuclock_base;
1174 	} while (tvp->tv_sec != gd->gd_time_seconds);
1175 
1176 	if (delta >= sys_cputimer->freq) {
1177 		tvp->tv_sec += delta / sys_cputimer->freq;
1178 		delta %= sys_cputimer->freq;
1179 	}
1180 	tvp->tv_usec = (sys_cputimer->freq64_usec * delta) >> 32;
1181 	if (tvp->tv_usec >= 1000000) {
1182 		tvp->tv_usec -= 1000000;
1183 		++tvp->tv_sec;
1184 	}
1185 }
1186 
1187 void
1188 getnanouptime(struct timespec *tsp)
1189 {
1190 	struct globaldata *gd = mycpu;
1191 	sysclock_t delta;
1192 
1193 	do {
1194 		tsp->tv_sec = gd->gd_time_seconds;
1195 		delta = gd->gd_hardclock.time - gd->gd_cpuclock_base;
1196 	} while (tsp->tv_sec != gd->gd_time_seconds);
1197 
1198 	if (delta >= sys_cputimer->freq) {
1199 		tsp->tv_sec += delta / sys_cputimer->freq;
1200 		delta %= sys_cputimer->freq;
1201 	}
1202 	tsp->tv_nsec = (sys_cputimer->freq64_nsec * delta) >> 32;
1203 }
1204 
1205 void
1206 microuptime(struct timeval *tvp)
1207 {
1208 	struct globaldata *gd = mycpu;
1209 	sysclock_t delta;
1210 
1211 	do {
1212 		tvp->tv_sec = gd->gd_time_seconds;
1213 		delta = sys_cputimer->count() - gd->gd_cpuclock_base;
1214 	} while (tvp->tv_sec != gd->gd_time_seconds);
1215 
1216 	if (delta >= sys_cputimer->freq) {
1217 		tvp->tv_sec += delta / sys_cputimer->freq;
1218 		delta %= sys_cputimer->freq;
1219 	}
1220 	tvp->tv_usec = (sys_cputimer->freq64_usec * delta) >> 32;
1221 }
1222 
1223 void
1224 nanouptime(struct timespec *tsp)
1225 {
1226 	struct globaldata *gd = mycpu;
1227 	sysclock_t delta;
1228 
1229 	do {
1230 		tsp->tv_sec = gd->gd_time_seconds;
1231 		delta = sys_cputimer->count() - gd->gd_cpuclock_base;
1232 	} while (tsp->tv_sec != gd->gd_time_seconds);
1233 
1234 	if (delta >= sys_cputimer->freq) {
1235 		tsp->tv_sec += delta / sys_cputimer->freq;
1236 		delta %= sys_cputimer->freq;
1237 	}
1238 	tsp->tv_nsec = (sys_cputimer->freq64_nsec * delta) >> 32;
1239 }
1240 
1241 /*
1242  * realtime routines
1243  */
1244 void
1245 getmicrotime(struct timeval *tvp)
1246 {
1247 	struct globaldata *gd = mycpu;
1248 	struct timespec *bt;
1249 	sysclock_t delta;
1250 
1251 	do {
1252 		tvp->tv_sec = gd->gd_time_seconds;
1253 		delta = gd->gd_hardclock.time - gd->gd_cpuclock_base;
1254 	} while (tvp->tv_sec != gd->gd_time_seconds);
1255 
1256 	if (delta >= sys_cputimer->freq) {
1257 		tvp->tv_sec += delta / sys_cputimer->freq;
1258 		delta %= sys_cputimer->freq;
1259 	}
1260 	tvp->tv_usec = (sys_cputimer->freq64_usec * delta) >> 32;
1261 
1262 	bt = &basetime[basetime_index];
1263 	cpu_lfence();
1264 	tvp->tv_sec += bt->tv_sec;
1265 	tvp->tv_usec += bt->tv_nsec / 1000;
1266 	while (tvp->tv_usec >= 1000000) {
1267 		tvp->tv_usec -= 1000000;
1268 		++tvp->tv_sec;
1269 	}
1270 }
1271 
1272 void
1273 getnanotime(struct timespec *tsp)
1274 {
1275 	struct globaldata *gd = mycpu;
1276 	struct timespec *bt;
1277 	sysclock_t delta;
1278 
1279 	do {
1280 		tsp->tv_sec = gd->gd_time_seconds;
1281 		delta = gd->gd_hardclock.time - gd->gd_cpuclock_base;
1282 	} while (tsp->tv_sec != gd->gd_time_seconds);
1283 
1284 	if (delta >= sys_cputimer->freq) {
1285 		tsp->tv_sec += delta / sys_cputimer->freq;
1286 		delta %= sys_cputimer->freq;
1287 	}
1288 	tsp->tv_nsec = (sys_cputimer->freq64_nsec * delta) >> 32;
1289 
1290 	bt = &basetime[basetime_index];
1291 	cpu_lfence();
1292 	tsp->tv_sec += bt->tv_sec;
1293 	tsp->tv_nsec += bt->tv_nsec;
1294 	while (tsp->tv_nsec >= 1000000000) {
1295 		tsp->tv_nsec -= 1000000000;
1296 		++tsp->tv_sec;
1297 	}
1298 }
1299 
1300 static void
1301 getnanotime_nbt(struct timespec *nbt, struct timespec *tsp)
1302 {
1303 	struct globaldata *gd = mycpu;
1304 	sysclock_t delta;
1305 
1306 	do {
1307 		tsp->tv_sec = gd->gd_time_seconds;
1308 		delta = gd->gd_hardclock.time - gd->gd_cpuclock_base;
1309 	} while (tsp->tv_sec != gd->gd_time_seconds);
1310 
1311 	if (delta >= sys_cputimer->freq) {
1312 		tsp->tv_sec += delta / sys_cputimer->freq;
1313 		delta %= sys_cputimer->freq;
1314 	}
1315 	tsp->tv_nsec = (sys_cputimer->freq64_nsec * delta) >> 32;
1316 
1317 	tsp->tv_sec += nbt->tv_sec;
1318 	tsp->tv_nsec += nbt->tv_nsec;
1319 	while (tsp->tv_nsec >= 1000000000) {
1320 		tsp->tv_nsec -= 1000000000;
1321 		++tsp->tv_sec;
1322 	}
1323 }
1324 
1325 
1326 void
1327 microtime(struct timeval *tvp)
1328 {
1329 	struct globaldata *gd = mycpu;
1330 	struct timespec *bt;
1331 	sysclock_t delta;
1332 
1333 	do {
1334 		tvp->tv_sec = gd->gd_time_seconds;
1335 		delta = sys_cputimer->count() - gd->gd_cpuclock_base;
1336 	} while (tvp->tv_sec != gd->gd_time_seconds);
1337 
1338 	if (delta >= sys_cputimer->freq) {
1339 		tvp->tv_sec += delta / sys_cputimer->freq;
1340 		delta %= sys_cputimer->freq;
1341 	}
1342 	tvp->tv_usec = (sys_cputimer->freq64_usec * delta) >> 32;
1343 
1344 	bt = &basetime[basetime_index];
1345 	cpu_lfence();
1346 	tvp->tv_sec += bt->tv_sec;
1347 	tvp->tv_usec += bt->tv_nsec / 1000;
1348 	while (tvp->tv_usec >= 1000000) {
1349 		tvp->tv_usec -= 1000000;
1350 		++tvp->tv_sec;
1351 	}
1352 }
1353 
1354 void
1355 nanotime(struct timespec *tsp)
1356 {
1357 	struct globaldata *gd = mycpu;
1358 	struct timespec *bt;
1359 	sysclock_t delta;
1360 
1361 	do {
1362 		tsp->tv_sec = gd->gd_time_seconds;
1363 		delta = sys_cputimer->count() - gd->gd_cpuclock_base;
1364 	} while (tsp->tv_sec != gd->gd_time_seconds);
1365 
1366 	if (delta >= sys_cputimer->freq) {
1367 		tsp->tv_sec += delta / sys_cputimer->freq;
1368 		delta %= sys_cputimer->freq;
1369 	}
1370 	tsp->tv_nsec = (sys_cputimer->freq64_nsec * delta) >> 32;
1371 
1372 	bt = &basetime[basetime_index];
1373 	cpu_lfence();
1374 	tsp->tv_sec += bt->tv_sec;
1375 	tsp->tv_nsec += bt->tv_nsec;
1376 	while (tsp->tv_nsec >= 1000000000) {
1377 		tsp->tv_nsec -= 1000000000;
1378 		++tsp->tv_sec;
1379 	}
1380 }
1381 
1382 /*
1383  * Get an approximate time_t.  It does not have to be accurate.  This
1384  * function is called only from KTR and can be called with the system in
1385  * any state so do not use a critical section or other complex operation
1386  * here.
1387  *
1388  * NOTE: This is not exactly synchronized with real time.  To do that we
1389  *	 would have to do what microtime does and check for a nanoseconds
1390  *	 overflow.
1391  */
1392 time_t
1393 get_approximate_time_t(void)
1394 {
1395 	struct globaldata *gd = mycpu;
1396 	struct timespec *bt;
1397 
1398 	bt = &basetime[basetime_index];
1399 	return(gd->gd_time_seconds + bt->tv_sec);
1400 }
1401 
1402 int
1403 pps_ioctl(u_long cmd, caddr_t data, struct pps_state *pps)
1404 {
1405 	pps_params_t *app;
1406 	struct pps_fetch_args *fapi;
1407 #ifdef PPS_SYNC
1408 	struct pps_kcbind_args *kapi;
1409 #endif
1410 
1411 	switch (cmd) {
1412 	case PPS_IOC_CREATE:
1413 		return (0);
1414 	case PPS_IOC_DESTROY:
1415 		return (0);
1416 	case PPS_IOC_SETPARAMS:
1417 		app = (pps_params_t *)data;
1418 		if (app->mode & ~pps->ppscap)
1419 			return (EINVAL);
1420 		pps->ppsparam = *app;
1421 		return (0);
1422 	case PPS_IOC_GETPARAMS:
1423 		app = (pps_params_t *)data;
1424 		*app = pps->ppsparam;
1425 		app->api_version = PPS_API_VERS_1;
1426 		return (0);
1427 	case PPS_IOC_GETCAP:
1428 		*(int*)data = pps->ppscap;
1429 		return (0);
1430 	case PPS_IOC_FETCH:
1431 		fapi = (struct pps_fetch_args *)data;
1432 		if (fapi->tsformat && fapi->tsformat != PPS_TSFMT_TSPEC)
1433 			return (EINVAL);
1434 		if (fapi->timeout.tv_sec || fapi->timeout.tv_nsec)
1435 			return (EOPNOTSUPP);
1436 		pps->ppsinfo.current_mode = pps->ppsparam.mode;
1437 		fapi->pps_info_buf = pps->ppsinfo;
1438 		return (0);
1439 	case PPS_IOC_KCBIND:
1440 #ifdef PPS_SYNC
1441 		kapi = (struct pps_kcbind_args *)data;
1442 		/* XXX Only root should be able to do this */
1443 		if (kapi->tsformat && kapi->tsformat != PPS_TSFMT_TSPEC)
1444 			return (EINVAL);
1445 		if (kapi->kernel_consumer != PPS_KC_HARDPPS)
1446 			return (EINVAL);
1447 		if (kapi->edge & ~pps->ppscap)
1448 			return (EINVAL);
1449 		pps->kcmode = kapi->edge;
1450 		return (0);
1451 #else
1452 		return (EOPNOTSUPP);
1453 #endif
1454 	default:
1455 		return (ENOTTY);
1456 	}
1457 }
1458 
1459 void
1460 pps_init(struct pps_state *pps)
1461 {
1462 	pps->ppscap |= PPS_TSFMT_TSPEC;
1463 	if (pps->ppscap & PPS_CAPTUREASSERT)
1464 		pps->ppscap |= PPS_OFFSETASSERT;
1465 	if (pps->ppscap & PPS_CAPTURECLEAR)
1466 		pps->ppscap |= PPS_OFFSETCLEAR;
1467 }
1468 
1469 void
1470 pps_event(struct pps_state *pps, sysclock_t count, int event)
1471 {
1472 	struct globaldata *gd;
1473 	struct timespec *tsp;
1474 	struct timespec *osp;
1475 	struct timespec *bt;
1476 	struct timespec ts;
1477 	sysclock_t *pcount;
1478 #ifdef PPS_SYNC
1479 	sysclock_t tcount;
1480 #endif
1481 	sysclock_t delta;
1482 	pps_seq_t *pseq;
1483 	int foff;
1484 #ifdef PPS_SYNC
1485 	int fhard;
1486 #endif
1487 	int ni;
1488 
1489 	gd = mycpu;
1490 
1491 	/* Things would be easier with arrays... */
1492 	if (event == PPS_CAPTUREASSERT) {
1493 		tsp = &pps->ppsinfo.assert_timestamp;
1494 		osp = &pps->ppsparam.assert_offset;
1495 		foff = pps->ppsparam.mode & PPS_OFFSETASSERT;
1496 #ifdef PPS_SYNC
1497 		fhard = pps->kcmode & PPS_CAPTUREASSERT;
1498 #endif
1499 		pcount = &pps->ppscount[0];
1500 		pseq = &pps->ppsinfo.assert_sequence;
1501 	} else {
1502 		tsp = &pps->ppsinfo.clear_timestamp;
1503 		osp = &pps->ppsparam.clear_offset;
1504 		foff = pps->ppsparam.mode & PPS_OFFSETCLEAR;
1505 #ifdef PPS_SYNC
1506 		fhard = pps->kcmode & PPS_CAPTURECLEAR;
1507 #endif
1508 		pcount = &pps->ppscount[1];
1509 		pseq = &pps->ppsinfo.clear_sequence;
1510 	}
1511 
1512 	/* Nothing really happened */
1513 	if (*pcount == count)
1514 		return;
1515 
1516 	*pcount = count;
1517 
1518 	do {
1519 		ts.tv_sec = gd->gd_time_seconds;
1520 		delta = count - gd->gd_cpuclock_base;
1521 	} while (ts.tv_sec != gd->gd_time_seconds);
1522 
1523 	if (delta >= sys_cputimer->freq) {
1524 		ts.tv_sec += delta / sys_cputimer->freq;
1525 		delta %= sys_cputimer->freq;
1526 	}
1527 	ts.tv_nsec = (sys_cputimer->freq64_nsec * delta) >> 32;
1528 	ni = basetime_index;
1529 	cpu_lfence();
1530 	bt = &basetime[ni];
1531 	ts.tv_sec += bt->tv_sec;
1532 	ts.tv_nsec += bt->tv_nsec;
1533 	while (ts.tv_nsec >= 1000000000) {
1534 		ts.tv_nsec -= 1000000000;
1535 		++ts.tv_sec;
1536 	}
1537 
1538 	(*pseq)++;
1539 	*tsp = ts;
1540 
1541 	if (foff) {
1542 		timespecadd(tsp, osp);
1543 		if (tsp->tv_nsec < 0) {
1544 			tsp->tv_nsec += 1000000000;
1545 			tsp->tv_sec -= 1;
1546 		}
1547 	}
1548 #ifdef PPS_SYNC
1549 	if (fhard) {
1550 		/* magic, at its best... */
1551 		tcount = count - pps->ppscount[2];
1552 		pps->ppscount[2] = count;
1553 		if (tcount >= sys_cputimer->freq) {
1554 			delta = (1000000000 * (tcount / sys_cputimer->freq) +
1555 				 sys_cputimer->freq64_nsec *
1556 				 (tcount % sys_cputimer->freq)) >> 32;
1557 		} else {
1558 			delta = (sys_cputimer->freq64_nsec * tcount) >> 32;
1559 		}
1560 		hardpps(tsp, delta);
1561 	}
1562 #endif
1563 }
1564 
1565 /*
1566  * Return the tsc target value for a delay of (ns).
1567  *
1568  * Returns -1 if the TSC is not supported.
1569  */
1570 int64_t
1571 tsc_get_target(int ns)
1572 {
1573 #if defined(_RDTSC_SUPPORTED_)
1574 	if (cpu_feature & CPUID_TSC) {
1575 		return (rdtsc() + tsc_frequency * ns / (int64_t)1000000000);
1576 	}
1577 #endif
1578 	return(-1);
1579 }
1580 
1581 /*
1582  * Compare the tsc against the passed target
1583  *
1584  * Returns +1 if the target has been reached
1585  * Returns  0 if the target has not yet been reached
1586  * Returns -1 if the TSC is not supported.
1587  *
1588  * Typical use:		while (tsc_test_target(target) == 0) { ...poll... }
1589  */
1590 int
1591 tsc_test_target(int64_t target)
1592 {
1593 #if defined(_RDTSC_SUPPORTED_)
1594 	if (cpu_feature & CPUID_TSC) {
1595 		if ((int64_t)(target - rdtsc()) <= 0)
1596 			return(1);
1597 		return(0);
1598 	}
1599 #endif
1600 	return(-1);
1601 }
1602 
1603 /*
1604  * Delay the specified number of nanoseconds using the tsc.  This function
1605  * returns immediately if the TSC is not supported.  At least one cpu_pause()
1606  * will be issued.
1607  */
1608 void
1609 tsc_delay(int ns)
1610 {
1611 	int64_t clk;
1612 
1613 	clk = tsc_get_target(ns);
1614 	cpu_pause();
1615 	while (tsc_test_target(clk) == 0)
1616 		cpu_pause();
1617 }
1618