xref: /openbsd-src/sys/kern/kern_clockintr.c (revision 7cfaf107872ed22ae1f9c20327bd5bf2b452848a)
1 /* $OpenBSD: kern_clockintr.c,v 1.49 2023/09/14 19:51:17 cheloha Exp $ */
2 /*
3  * Copyright (c) 2003 Dale Rahn <drahn@openbsd.org>
4  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
5  * Copyright (c) 2020-2022 Scott Cheloha <cheloha@openbsd.org>
6  *
7  * Permission to use, copy, modify, and distribute this software for any
8  * purpose with or without fee is hereby granted, provided that the above
9  * copyright notice and this permission notice appear in all copies.
10  *
11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18  */
19 
20 #include <sys/param.h>
21 #include <sys/systm.h>
22 #include <sys/atomic.h>
23 #include <sys/clockintr.h>
24 #include <sys/kernel.h>
25 #include <sys/malloc.h>
26 #include <sys/mutex.h>
27 #include <sys/resourcevar.h>
28 #include <sys/queue.h>
29 #include <sys/sched.h>
30 #include <sys/stdint.h>
31 #include <sys/sysctl.h>
32 #include <sys/time.h>
33 
34 /*
35  * Protection for global variables in this file:
36  *
37  *	I	Immutable after initialization.
38  */
39 uint32_t clockintr_flags;		/* [I] global state + behavior flags */
40 uint32_t hardclock_period;		/* [I] hardclock period (ns) */
41 uint32_t statclock_avg;			/* [I] average statclock period (ns) */
42 uint32_t statclock_min;			/* [I] minimum statclock period (ns) */
43 uint32_t statclock_mask;		/* [I] set of allowed offsets */
44 
45 void clockintr_hardclock(struct clockintr *, void *, void *);
46 void clockintr_schedule(struct clockintr *, uint64_t);
47 void clockintr_schedule_locked(struct clockintr *, uint64_t);
48 void clockintr_statclock(struct clockintr *, void *, void *);
49 void clockqueue_intrclock_install(struct clockintr_queue *,
50     const struct intrclock *);
51 uint64_t clockqueue_next(const struct clockintr_queue *);
52 void clockqueue_pend_delete(struct clockintr_queue *, struct clockintr *);
53 void clockqueue_pend_insert(struct clockintr_queue *, struct clockintr *,
54     uint64_t);
55 void clockqueue_reset_intrclock(struct clockintr_queue *);
56 uint64_t nsec_advance(uint64_t *, uint64_t, uint64_t);
57 
58 /*
59  * Initialize global state.  Set flags and compute intervals.
60  */
61 void
62 clockintr_init(uint32_t flags)
63 {
64 	uint32_t half_avg, var;
65 
66 	KASSERT(CPU_IS_PRIMARY(curcpu()));
67 	KASSERT(clockintr_flags == 0);
68 	KASSERT(!ISSET(flags, ~CL_FLAG_MASK));
69 
70 	KASSERT(hz > 0 && hz <= 1000000000);
71 	hardclock_period = 1000000000 / hz;
72 	roundrobin_period = hardclock_period * 10;
73 
74 	KASSERT(stathz >= 1 && stathz <= 1000000000);
75 
76 	/*
77 	 * Compute the average statclock() period.  Then find var, the
78 	 * largest power of two such that var <= statclock_avg / 2.
79 	 */
80 	statclock_avg = 1000000000 / stathz;
81 	half_avg = statclock_avg / 2;
82 	for (var = 1U << 31; var > half_avg; var /= 2)
83 		continue;
84 
85 	/*
86 	 * Set a lower bound for the range using statclock_avg and var.
87 	 * The mask for that range is just (var - 1).
88 	 */
89 	statclock_min = statclock_avg - (var / 2);
90 	statclock_mask = var - 1;
91 
92 	SET(clockintr_flags, flags | CL_INIT);
93 }
94 
95 /*
96  * Ready the calling CPU for clockintr_dispatch().  If this is our
97  * first time here, install the intrclock, if any, and set necessary
98  * flags.  Advance the schedule as needed.
99  */
100 void
101 clockintr_cpu_init(const struct intrclock *ic)
102 {
103 	uint64_t multiplier = 0;
104 	struct cpu_info *ci = curcpu();
105 	struct clockintr_queue *cq = &ci->ci_queue;
106 	struct schedstate_percpu *spc = &ci->ci_schedstate;
107 	int reset_cq_intrclock = 0;
108 
109 	KASSERT(ISSET(clockintr_flags, CL_INIT));
110 
111 	if (ic != NULL)
112 		clockqueue_intrclock_install(cq, ic);
113 
114 	/* TODO: Remove these from struct clockintr_queue. */
115 	if (cq->cq_hardclock == NULL) {
116 		cq->cq_hardclock = clockintr_establish(ci, clockintr_hardclock,
117 		    NULL);
118 		if (cq->cq_hardclock == NULL)
119 			panic("%s: failed to establish hardclock", __func__);
120 	}
121 	if (cq->cq_statclock == NULL) {
122 		cq->cq_statclock = clockintr_establish(ci, clockintr_statclock,
123 		    NULL);
124 		if (cq->cq_statclock == NULL)
125 			panic("%s: failed to establish statclock", __func__);
126 	}
127 
128 	/*
129 	 * Mask CQ_INTRCLOCK while we're advancing the internal clock
130 	 * interrupts.  We don't want the intrclock to fire until this
131 	 * thread reaches clockintr_trigger().
132 	 */
133 	if (ISSET(cq->cq_flags, CQ_INTRCLOCK)) {
134 		CLR(cq->cq_flags, CQ_INTRCLOCK);
135 		reset_cq_intrclock = 1;
136 	}
137 
138 	/*
139 	 * Until we understand scheduler lock contention better, stagger
140 	 * the hardclock and statclock so they don't all happen at once.
141 	 * If we have no intrclock it doesn't matter, we have no control
142 	 * anyway.  The primary CPU's starting offset is always zero, so
143 	 * leave the multiplier zero.
144 	 */
145 	if (!CPU_IS_PRIMARY(ci) && reset_cq_intrclock)
146 		multiplier = CPU_INFO_UNIT(ci);
147 
148 	/*
149 	 * The first time we do this, the primary CPU cannot skip any
150 	 * hardclocks.  We can skip hardclocks on subsequent calls because
151 	 * the global tick value is advanced during inittodr(9) on our
152 	 * behalf.
153 	 */
154 	if (CPU_IS_PRIMARY(ci)) {
155 		if (cq->cq_hardclock->cl_expiration == 0)
156 			clockintr_schedule(cq->cq_hardclock, 0);
157 		else
158 			clockintr_advance(cq->cq_hardclock, hardclock_period);
159 	} else {
160 		if (cq->cq_hardclock->cl_expiration == 0) {
161 			clockintr_stagger(cq->cq_hardclock, hardclock_period,
162 			     multiplier, MAXCPUS);
163 		}
164 		clockintr_advance(cq->cq_hardclock, hardclock_period);
165 	}
166 
167 	/*
168 	 * We can always advance the statclock.  There is no reason to
169 	 * stagger a randomized statclock.
170 	 */
171 	if (!statclock_is_randomized) {
172 		if (cq->cq_statclock->cl_expiration == 0) {
173 			clockintr_stagger(cq->cq_statclock, statclock_avg,
174 			    multiplier, MAXCPUS);
175 		}
176 	}
177 	clockintr_advance(cq->cq_statclock, statclock_avg);
178 
179 	/*
180 	 * XXX Need to find a better place to do this.  We can't do it in
181 	 * sched_init_cpu() because initclocks() runs after it.
182 	 */
183 	if (spc->spc_itimer->cl_expiration == 0) {
184 		clockintr_stagger(spc->spc_itimer, hardclock_period,
185 		    multiplier, MAXCPUS);
186 	}
187 	if (spc->spc_profclock->cl_expiration == 0) {
188 		clockintr_stagger(spc->spc_profclock, profclock_period,
189 		    multiplier, MAXCPUS);
190 	}
191 	if (spc->spc_roundrobin->cl_expiration == 0) {
192 		clockintr_stagger(spc->spc_roundrobin, hardclock_period,
193 		    multiplier, MAXCPUS);
194 	}
195 	clockintr_advance(spc->spc_roundrobin, roundrobin_period);
196 
197 	if (reset_cq_intrclock)
198 		SET(cq->cq_flags, CQ_INTRCLOCK);
199 }
200 
201 /*
202  * If we have an intrclock, trigger it to start the dispatch cycle.
203  */
204 void
205 clockintr_trigger(void)
206 {
207 	struct clockintr_queue *cq = &curcpu()->ci_queue;
208 
209 	KASSERT(ISSET(cq->cq_flags, CQ_INIT));
210 
211 	if (ISSET(cq->cq_flags, CQ_INTRCLOCK))
212 		intrclock_trigger(&cq->cq_intrclock);
213 }
214 
215 /*
216  * Run all expired events scheduled on the calling CPU.
217  */
218 int
219 clockintr_dispatch(void *frame)
220 {
221 	uint64_t lateness, run = 0, start;
222 	struct cpu_info *ci = curcpu();
223 	struct clockintr *cl, *shadow;
224 	struct clockintr_queue *cq = &ci->ci_queue;
225 	uint32_t ogen;
226 
227 	if (cq->cq_dispatch != 0)
228 		panic("%s: recursive dispatch", __func__);
229 	cq->cq_dispatch = 1;
230 
231 	splassert(IPL_CLOCK);
232 	KASSERT(ISSET(cq->cq_flags, CQ_INIT));
233 
234 	mtx_enter(&cq->cq_mtx);
235 
236 	/*
237 	 * If nothing is scheduled or we arrived too early, we have
238 	 * nothing to do.
239 	 */
240 	start = nsecuptime();
241 	cq->cq_uptime = start;
242 	if (TAILQ_EMPTY(&cq->cq_pend))
243 		goto stats;
244 	if (cq->cq_uptime < clockqueue_next(cq))
245 		goto rearm;
246 	lateness = start - clockqueue_next(cq);
247 
248 	/*
249 	 * Dispatch expired events.
250 	 */
251 	for (;;) {
252 		cl = TAILQ_FIRST(&cq->cq_pend);
253 		if (cl == NULL)
254 			break;
255 		if (cq->cq_uptime < cl->cl_expiration) {
256 			/* Double-check the time before giving up. */
257 			cq->cq_uptime = nsecuptime();
258 			if (cq->cq_uptime < cl->cl_expiration)
259 				break;
260 		}
261 
262 		/*
263 		 * This clockintr has expired.  Initialize a shadow copy
264 		 * and execute it.
265 		 */
266 		clockqueue_pend_delete(cq, cl);
267 		shadow = &cq->cq_shadow;
268 		shadow->cl_expiration = cl->cl_expiration;
269 		shadow->cl_arg = cl->cl_arg;
270 		shadow->cl_func = cl->cl_func;
271 		cq->cq_running = cl;
272 		mtx_leave(&cq->cq_mtx);
273 
274 		shadow->cl_func(shadow, frame, shadow->cl_arg);
275 
276 		mtx_enter(&cq->cq_mtx);
277 		cq->cq_running = NULL;
278 		if (ISSET(cl->cl_flags, CLST_IGNORE_SHADOW)) {
279 			CLR(cl->cl_flags, CLST_IGNORE_SHADOW);
280 			CLR(shadow->cl_flags, CLST_SHADOW_PENDING);
281 		}
282 		if (ISSET(shadow->cl_flags, CLST_SHADOW_PENDING)) {
283 			CLR(shadow->cl_flags, CLST_SHADOW_PENDING);
284 			clockqueue_pend_insert(cq, cl, shadow->cl_expiration);
285 		}
286 		run++;
287 	}
288 
289 	/*
290 	 * Dispatch complete.
291 	 */
292 rearm:
293 	/* Rearm the interrupt clock if we have one. */
294 	if (ISSET(cq->cq_flags, CQ_INTRCLOCK)) {
295 		if (!TAILQ_EMPTY(&cq->cq_pend)) {
296 			intrclock_rearm(&cq->cq_intrclock,
297 			    clockqueue_next(cq) - cq->cq_uptime);
298 		}
299 	}
300 stats:
301 	/* Update our stats. */
302 	ogen = cq->cq_gen;
303 	cq->cq_gen = 0;
304 	membar_producer();
305 	cq->cq_stat.cs_dispatched += cq->cq_uptime - start;
306 	if (run > 0) {
307 		cq->cq_stat.cs_lateness += lateness;
308 		cq->cq_stat.cs_prompt++;
309 		cq->cq_stat.cs_run += run;
310 	} else if (!TAILQ_EMPTY(&cq->cq_pend)) {
311 		cq->cq_stat.cs_early++;
312 		cq->cq_stat.cs_earliness += clockqueue_next(cq) - cq->cq_uptime;
313 	} else
314 		cq->cq_stat.cs_spurious++;
315 	membar_producer();
316 	cq->cq_gen = MAX(1, ogen + 1);
317 
318 	mtx_leave(&cq->cq_mtx);
319 
320 	if (cq->cq_dispatch != 1)
321 		panic("%s: unexpected value: %u", __func__, cq->cq_dispatch);
322 	cq->cq_dispatch = 0;
323 
324 	return run > 0;
325 }
326 
327 uint64_t
328 clockintr_advance(struct clockintr *cl, uint64_t period)
329 {
330 	uint64_t count, expiration;
331 	struct clockintr_queue *cq = cl->cl_queue;
332 
333 	if (cl == &cq->cq_shadow) {
334 		count = nsec_advance(&cl->cl_expiration, period, cq->cq_uptime);
335 		SET(cl->cl_flags, CLST_SHADOW_PENDING);
336 	} else {
337 		mtx_enter(&cq->cq_mtx);
338 		expiration = cl->cl_expiration;
339 		count = nsec_advance(&expiration, period, nsecuptime());
340 		clockintr_schedule_locked(cl, expiration);
341 		mtx_leave(&cq->cq_mtx);
342 	}
343 	return count;
344 }
345 
346 uint64_t
347 clockintr_advance_random(struct clockintr *cl, uint64_t min, uint32_t mask)
348 {
349 	uint64_t count = 0;
350 	struct clockintr_queue *cq = cl->cl_queue;
351 	uint32_t off;
352 
353 	KASSERT(cl == &cq->cq_shadow);
354 
355 	while (cl->cl_expiration <= cq->cq_uptime) {
356 		while ((off = (random() & mask)) == 0)
357 			continue;
358 		cl->cl_expiration += min + off;
359 		count++;
360 	}
361 	SET(cl->cl_flags, CLST_SHADOW_PENDING);
362 	return count;
363 }
364 
365 void
366 clockintr_cancel(struct clockintr *cl)
367 {
368 	struct clockintr_queue *cq = cl->cl_queue;
369 	int was_next;
370 
371 	if (cl == &cq->cq_shadow) {
372 		CLR(cl->cl_flags, CLST_SHADOW_PENDING);
373 		return;
374 	}
375 
376 	mtx_enter(&cq->cq_mtx);
377 	if (ISSET(cl->cl_flags, CLST_PENDING)) {
378 		was_next = cl == TAILQ_FIRST(&cq->cq_pend);
379 		clockqueue_pend_delete(cq, cl);
380 		if (ISSET(cq->cq_flags, CQ_INTRCLOCK)) {
381 			if (was_next && !TAILQ_EMPTY(&cq->cq_pend)) {
382 				if (cq == &curcpu()->ci_queue)
383 					clockqueue_reset_intrclock(cq);
384 			}
385 		}
386 	}
387 	if (cl == cq->cq_running)
388 		SET(cl->cl_flags, CLST_IGNORE_SHADOW);
389 	mtx_leave(&cq->cq_mtx);
390 }
391 
392 struct clockintr *
393 clockintr_establish(struct cpu_info *ci,
394     void (*func)(struct clockintr *, void *, void *), void *arg)
395 {
396 	struct clockintr *cl;
397 	struct clockintr_queue *cq = &ci->ci_queue;
398 
399 	cl = malloc(sizeof *cl, M_DEVBUF, M_NOWAIT | M_ZERO);
400 	if (cl == NULL)
401 		return NULL;
402 	cl->cl_arg = arg;
403 	cl->cl_func = func;
404 	cl->cl_queue = cq;
405 
406 	mtx_enter(&cq->cq_mtx);
407 	TAILQ_INSERT_TAIL(&cq->cq_est, cl, cl_elink);
408 	mtx_leave(&cq->cq_mtx);
409 	return cl;
410 }
411 
412 void
413 clockintr_schedule(struct clockintr *cl, uint64_t expiration)
414 {
415 	struct clockintr_queue *cq = cl->cl_queue;
416 
417 	if (cl == &cq->cq_shadow) {
418 		cl->cl_expiration = expiration;
419 		SET(cl->cl_flags, CLST_SHADOW_PENDING);
420 	} else {
421 		mtx_enter(&cq->cq_mtx);
422 		clockintr_schedule_locked(cl, expiration);
423 		mtx_leave(&cq->cq_mtx);
424 	}
425 }
426 
427 void
428 clockintr_schedule_locked(struct clockintr *cl, uint64_t expiration)
429 {
430 	struct clockintr_queue *cq = cl->cl_queue;
431 
432 	MUTEX_ASSERT_LOCKED(&cq->cq_mtx);
433 
434 	if (ISSET(cl->cl_flags, CLST_PENDING))
435 		clockqueue_pend_delete(cq, cl);
436 	clockqueue_pend_insert(cq, cl, expiration);
437 	if (ISSET(cq->cq_flags, CQ_INTRCLOCK)) {
438 		if (cl == TAILQ_FIRST(&cq->cq_pend)) {
439 			if (cq == &curcpu()->ci_queue)
440 				clockqueue_reset_intrclock(cq);
441 		}
442 	}
443 	if (cl == cq->cq_running)
444 		SET(cl->cl_flags, CLST_IGNORE_SHADOW);
445 }
446 
447 void
448 clockintr_stagger(struct clockintr *cl, uint64_t period, uint32_t n,
449     uint32_t count)
450 {
451 	struct clockintr_queue *cq = cl->cl_queue;
452 
453 	KASSERT(n < count);
454 
455 	mtx_enter(&cq->cq_mtx);
456 	if (ISSET(cl->cl_flags, CLST_PENDING))
457 		panic("%s: clock interrupt pending", __func__);
458 	cl->cl_expiration = period / count * n;
459 	mtx_leave(&cq->cq_mtx);
460 }
461 
462 void
463 clockintr_hardclock(struct clockintr *cl, void *frame, void *arg)
464 {
465 	uint64_t count, i;
466 
467 	count = clockintr_advance(cl, hardclock_period);
468 	for (i = 0; i < count; i++)
469 		hardclock(frame);
470 }
471 
472 void
473 clockintr_statclock(struct clockintr *cl, void *frame, void *arg)
474 {
475 	uint64_t count, i;
476 
477 	if (statclock_is_randomized) {
478 		count = clockintr_advance_random(cl, statclock_min,
479 		    statclock_mask);
480 	} else {
481 		count = clockintr_advance(cl, statclock_avg);
482 	}
483 	for (i = 0; i < count; i++)
484 		statclock(frame);
485 }
486 
487 void
488 clockqueue_init(struct clockintr_queue *cq)
489 {
490 	if (ISSET(cq->cq_flags, CQ_INIT))
491 		return;
492 
493 	cq->cq_shadow.cl_queue = cq;
494 	mtx_init(&cq->cq_mtx, IPL_CLOCK);
495 	TAILQ_INIT(&cq->cq_est);
496 	TAILQ_INIT(&cq->cq_pend);
497 	cq->cq_gen = 1;
498 	SET(cq->cq_flags, CQ_INIT);
499 }
500 
501 void
502 clockqueue_intrclock_install(struct clockintr_queue *cq,
503     const struct intrclock *ic)
504 {
505 	mtx_enter(&cq->cq_mtx);
506 	if (!ISSET(cq->cq_flags, CQ_INTRCLOCK)) {
507 		cq->cq_intrclock = *ic;
508 		SET(cq->cq_flags, CQ_INTRCLOCK);
509 	}
510 	mtx_leave(&cq->cq_mtx);
511 }
512 
513 uint64_t
514 clockqueue_next(const struct clockintr_queue *cq)
515 {
516 	MUTEX_ASSERT_LOCKED(&cq->cq_mtx);
517 	return TAILQ_FIRST(&cq->cq_pend)->cl_expiration;
518 }
519 
520 void
521 clockqueue_pend_delete(struct clockintr_queue *cq, struct clockintr *cl)
522 {
523 	MUTEX_ASSERT_LOCKED(&cq->cq_mtx);
524 	KASSERT(ISSET(cl->cl_flags, CLST_PENDING));
525 
526 	TAILQ_REMOVE(&cq->cq_pend, cl, cl_plink);
527 	CLR(cl->cl_flags, CLST_PENDING);
528 }
529 
530 
531 void
532 clockqueue_pend_insert(struct clockintr_queue *cq, struct clockintr *cl,
533     uint64_t expiration)
534 {
535 	struct clockintr *elm;
536 
537 	MUTEX_ASSERT_LOCKED(&cq->cq_mtx);
538 	KASSERT(!ISSET(cl->cl_flags, CLST_PENDING));
539 
540 	cl->cl_expiration = expiration;
541 	TAILQ_FOREACH(elm, &cq->cq_pend, cl_plink) {
542 		if (cl->cl_expiration < elm->cl_expiration)
543 			break;
544 	}
545 	if (elm == NULL)
546 		TAILQ_INSERT_TAIL(&cq->cq_pend, cl, cl_plink);
547 	else
548 		TAILQ_INSERT_BEFORE(elm, cl, cl_plink);
549 	SET(cl->cl_flags, CLST_PENDING);
550 }
551 
552 void
553 clockqueue_reset_intrclock(struct clockintr_queue *cq)
554 {
555 	uint64_t exp, now;
556 
557 	MUTEX_ASSERT_LOCKED(&cq->cq_mtx);
558 	KASSERT(ISSET(cq->cq_flags, CQ_INTRCLOCK));
559 
560 	exp = clockqueue_next(cq);
561 	now = nsecuptime();
562 	if (now < exp)
563 		intrclock_rearm(&cq->cq_intrclock, exp - now);
564 	else
565 		intrclock_trigger(&cq->cq_intrclock);
566 }
567 
568 /*
569  * Advance *next in increments of period until it exceeds now.
570  * Returns the number of increments *next was advanced.
571  *
572  * We check the common cases first to avoid division if possible.
573  * This does no overflow checking.
574  */
575 uint64_t
576 nsec_advance(uint64_t *next, uint64_t period, uint64_t now)
577 {
578 	uint64_t elapsed;
579 
580 	if (now < *next)
581 		return 0;
582 
583 	if (now < *next + period) {
584 		*next += period;
585 		return 1;
586 	}
587 
588 	elapsed = (now - *next) / period + 1;
589 	*next += period * elapsed;
590 	return elapsed;
591 }
592 
593 int
594 sysctl_clockintr(int *name, u_int namelen, void *oldp, size_t *oldlenp,
595     void *newp, size_t newlen)
596 {
597 	struct clockintr_stat sum, tmp;
598 	struct clockintr_queue *cq;
599 	struct cpu_info *ci;
600 	CPU_INFO_ITERATOR cii;
601 	uint32_t gen;
602 
603 	if (namelen != 1)
604 		return ENOTDIR;
605 
606 	switch (name[0]) {
607 	case KERN_CLOCKINTR_STATS:
608 		memset(&sum, 0, sizeof sum);
609 		CPU_INFO_FOREACH(cii, ci) {
610 			cq = &ci->ci_queue;
611 			if (!ISSET(cq->cq_flags, CQ_INIT))
612 				continue;
613 			do {
614 				gen = cq->cq_gen;
615 				membar_consumer();
616 				tmp = cq->cq_stat;
617 				membar_consumer();
618 			} while (gen == 0 || gen != cq->cq_gen);
619 			sum.cs_dispatched += tmp.cs_dispatched;
620 			sum.cs_early += tmp.cs_early;
621 			sum.cs_earliness += tmp.cs_earliness;
622 			sum.cs_lateness += tmp.cs_lateness;
623 			sum.cs_prompt += tmp.cs_prompt;
624 			sum.cs_run += tmp.cs_run;
625 			sum.cs_spurious += tmp.cs_spurious;
626 		}
627 		return sysctl_rdstruct(oldp, oldlenp, newp, &sum, sizeof sum);
628 	default:
629 		break;
630 	}
631 
632 	return EINVAL;
633 }
634 
635 #ifdef DDB
636 
637 #include <machine/db_machdep.h>
638 
639 #include <ddb/db_interface.h>
640 #include <ddb/db_output.h>
641 #include <ddb/db_sym.h>
642 
643 void db_show_clockintr(const struct clockintr *, const char *, u_int);
644 void db_show_clockintr_cpu(struct cpu_info *);
645 
646 void
647 db_show_all_clockintr(db_expr_t addr, int haddr, db_expr_t count, char *modif)
648 {
649 	struct timespec now;
650 	struct cpu_info *ci;
651 	CPU_INFO_ITERATOR cii;
652 
653 	nanouptime(&now);
654 	db_printf("%20s\n", "UPTIME");
655 	db_printf("%10lld.%09ld\n", now.tv_sec, now.tv_nsec);
656 	db_printf("\n");
657 	db_printf("%20s  %5s  %3s  %s\n", "EXPIRATION", "STATE", "CPU", "NAME");
658 	CPU_INFO_FOREACH(cii, ci) {
659 		if (ISSET(ci->ci_queue.cq_flags, CQ_INIT))
660 			db_show_clockintr_cpu(ci);
661 	}
662 }
663 
664 void
665 db_show_clockintr_cpu(struct cpu_info *ci)
666 {
667 	struct clockintr *elm;
668 	struct clockintr_queue *cq = &ci->ci_queue;
669 	u_int cpu = CPU_INFO_UNIT(ci);
670 
671 	if (cq->cq_running != NULL)
672 		db_show_clockintr(cq->cq_running, "run", cpu);
673 	TAILQ_FOREACH(elm, &cq->cq_pend, cl_plink)
674 		db_show_clockintr(elm, "pend", cpu);
675 	TAILQ_FOREACH(elm, &cq->cq_est, cl_elink) {
676 		if (!ISSET(elm->cl_flags, CLST_PENDING))
677 			db_show_clockintr(elm, "idle", cpu);
678 	}
679 }
680 
681 void
682 db_show_clockintr(const struct clockintr *cl, const char *state, u_int cpu)
683 {
684 	struct timespec ts;
685 	char *name;
686 	db_expr_t offset;
687 
688 	NSEC_TO_TIMESPEC(cl->cl_expiration, &ts);
689 	db_find_sym_and_offset((vaddr_t)cl->cl_func, &name, &offset);
690 	if (name == NULL)
691 		name = "?";
692 	db_printf("%10lld.%09ld  %5s  %3u  %s\n",
693 	    ts.tv_sec, ts.tv_nsec, state, cpu, name);
694 }
695 
696 #endif /* DDB */
697