1*4c6ec8e8Sad /* $NetBSD: profile.c,v 1.11 2020/05/15 23:57:17 ad Exp $ */
201c9547eSdarran
3bb8023b5Sdarran /*
4bb8023b5Sdarran * CDDL HEADER START
5bb8023b5Sdarran *
6bb8023b5Sdarran * The contents of this file are subject to the terms of the
7bb8023b5Sdarran * Common Development and Distribution License (the "License").
8bb8023b5Sdarran * You may not use this file except in compliance with the License.
9bb8023b5Sdarran *
10bb8023b5Sdarran * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
11bb8023b5Sdarran * or http://www.opensolaris.org/os/licensing.
12bb8023b5Sdarran * See the License for the specific language governing permissions
13bb8023b5Sdarran * and limitations under the License.
14bb8023b5Sdarran *
15bb8023b5Sdarran * When distributing Covered Code, include this CDDL HEADER in each
16bb8023b5Sdarran * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
17bb8023b5Sdarran * If applicable, add the following below this CDDL HEADER, with the
18bb8023b5Sdarran * fields enclosed by brackets "[]" replaced with your own identifying
19bb8023b5Sdarran * information: Portions Copyright [yyyy] [name of copyright owner]
20bb8023b5Sdarran *
21bb8023b5Sdarran * CDDL HEADER END
22bb8023b5Sdarran *
23bb8023b5Sdarran * Portions Copyright 2006-2008 John Birrell jb@freebsd.org
24bb8023b5Sdarran *
25ba2539a9Schs * $FreeBSD: head/sys/cddl/dev/profile/profile.c 300618 2016-05-24 16:41:37Z br $
26bb8023b5Sdarran *
27bb8023b5Sdarran */
28bb8023b5Sdarran
29bb8023b5Sdarran /*
30bb8023b5Sdarran * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
31bb8023b5Sdarran * Use is subject to license terms.
32bb8023b5Sdarran */
33bb8023b5Sdarran
34bb8023b5Sdarran #include <sys/cdefs.h>
35bb8023b5Sdarran #include <sys/param.h>
36bb8023b5Sdarran #include <sys/systm.h>
37bb8023b5Sdarran #include <sys/conf.h>
38bb8023b5Sdarran #include <sys/cpuvar.h>
39bb8023b5Sdarran #include <sys/fcntl.h>
40bb8023b5Sdarran #include <sys/filio.h>
418f4ea3c4Schs #ifdef __FreeBSD__
42bb8023b5Sdarran #include <sys/kdb.h>
438f4ea3c4Schs #endif
44bb8023b5Sdarran #include <sys/kernel.h>
45bb8023b5Sdarran #include <sys/kmem.h>
46bb8023b5Sdarran #include <sys/kthread.h>
47ba2539a9Schs #ifdef __FreeBSD__
48ba2539a9Schs #include <sys/limits.h>
49ba2539a9Schs #endif
50bb8023b5Sdarran #include <sys/linker.h>
51bb8023b5Sdarran #include <sys/lock.h>
52bb8023b5Sdarran #include <sys/malloc.h>
53bb8023b5Sdarran #include <sys/module.h>
54bb8023b5Sdarran #include <sys/mutex.h>
55bb8023b5Sdarran #include <sys/poll.h>
56bb8023b5Sdarran #include <sys/proc.h>
57bb8023b5Sdarran #include <sys/selinfo.h>
588f4ea3c4Schs #ifdef __FreeBSD__
59bb8023b5Sdarran #include <sys/smp.h>
60ba2539a9Schs #include <sys/sysctl.h>
618f4ea3c4Schs #endif
62bb8023b5Sdarran #include <sys/uio.h>
63bb8023b5Sdarran #include <sys/unistd.h>
64ba2539a9Schs #ifdef __FreeBSD__
65ba2539a9Schs #include <machine/cpu.h>
66ba2539a9Schs #include <machine/stdarg.h>
678f4ea3c4Schs #endif
688f4ea3c4Schs
69ba2539a9Schs #ifdef __NetBSD__
70ba2539a9Schs #include <sys/syslimits.h>
71ba2539a9Schs #include <sys/atomic.h>
72ba2539a9Schs #include <sys/cpu.h>
73bb8023b5Sdarran #include <sys/cyclic.h>
74ba2539a9Schs #endif
75ba2539a9Schs
76bb8023b5Sdarran #include <sys/dtrace.h>
77bb8023b5Sdarran #include <sys/dtrace_bsd.h>
78bb8023b5Sdarran
79bb8023b5Sdarran #define PROF_NAMELEN 15
80bb8023b5Sdarran
81bb8023b5Sdarran #define PROF_PROFILE 0
82bb8023b5Sdarran #define PROF_TICK 1
83bb8023b5Sdarran #define PROF_PREFIX_PROFILE "profile-"
84bb8023b5Sdarran #define PROF_PREFIX_TICK "tick-"
85bb8023b5Sdarran
86bb8023b5Sdarran /*
87bb8023b5Sdarran * Regardless of platform, there are five artificial frames in the case of the
88bb8023b5Sdarran * profile provider:
89bb8023b5Sdarran *
90bb8023b5Sdarran * profile_fire
91bb8023b5Sdarran * cyclic_expire
92bb8023b5Sdarran * cyclic_fire
93bb8023b5Sdarran * [ cbe ]
94bb8023b5Sdarran * [ locore ]
95bb8023b5Sdarran *
96bb8023b5Sdarran * On amd64, there are two frames associated with locore: one in locore, and
97bb8023b5Sdarran * another in common interrupt dispatch code. (i386 has not been modified to
98bb8023b5Sdarran * use this common layer.) Further, on i386, the interrupted instruction
99bb8023b5Sdarran * appears as its own stack frame. All of this means that we need to add one
100bb8023b5Sdarran * frame for amd64, and then take one away for both amd64 and i386.
101bb8023b5Sdarran *
102bb8023b5Sdarran * On SPARC, the picture is further complicated because the compiler
103bb8023b5Sdarran * optimizes away tail-calls -- so the following frames are optimized away:
104bb8023b5Sdarran *
105bb8023b5Sdarran * profile_fire
106bb8023b5Sdarran * cyclic_expire
107bb8023b5Sdarran *
108bb8023b5Sdarran * This gives three frames. However, on DEBUG kernels, the cyclic_expire
109bb8023b5Sdarran * frame cannot be tail-call eliminated, yielding four frames in this case.
110bb8023b5Sdarran *
111bb8023b5Sdarran * All of the above constraints lead to the mess below. Yes, the profile
112bb8023b5Sdarran * provider should ideally figure this out on-the-fly by hiting one of its own
113bb8023b5Sdarran * probes and then walking its own stack trace. This is complicated, however,
114bb8023b5Sdarran * and the static definition doesn't seem to be overly brittle. Still, we
115bb8023b5Sdarran * allow for a manual override in case we get it completely wrong.
116bb8023b5Sdarran */
1178f4ea3c4Schs #ifdef __FreeBSD__
118bb8023b5Sdarran #ifdef __amd64
119ba2539a9Schs #define PROF_ARTIFICIAL_FRAMES 10
120bb8023b5Sdarran #else
121bb8023b5Sdarran #ifdef __i386
122bb8023b5Sdarran #define PROF_ARTIFICIAL_FRAMES 6
123bb8023b5Sdarran #else
124bb8023b5Sdarran #ifdef __sparc
125bb8023b5Sdarran #ifdef DEBUG
126bb8023b5Sdarran #define PROF_ARTIFICIAL_FRAMES 4
127bb8023b5Sdarran #else
128bb8023b5Sdarran #define PROF_ARTIFICIAL_FRAMES 3
129bb8023b5Sdarran #endif
130bb8023b5Sdarran #endif
131bb8023b5Sdarran #endif
132bb8023b5Sdarran #endif
133ba2539a9Schs
134ba2539a9Schs #ifdef __mips
135ba2539a9Schs /*
136ba2539a9Schs * This value is bogus just to make module compilable on mips
137ba2539a9Schs */
138ba2539a9Schs #define PROF_ARTIFICIAL_FRAMES 3
1398f4ea3c4Schs #endif
1408f4ea3c4Schs
141ba2539a9Schs #ifdef __powerpc__
142ba2539a9Schs /*
143ba2539a9Schs * This value is bogus just to make module compilable on powerpc
144ba2539a9Schs */
145ba2539a9Schs #define PROF_ARTIFICIAL_FRAMES 3
146ba2539a9Schs #endif
147ba2539a9Schs
148ba2539a9Schs struct profile_probe_percpu;
149ba2539a9Schs
150ba2539a9Schs #ifdef __mips
151ba2539a9Schs /* bogus */
152ba2539a9Schs #define PROF_ARTIFICIAL_FRAMES 3
153ba2539a9Schs #endif
154ba2539a9Schs
155ba2539a9Schs #ifdef __arm__
156ba2539a9Schs #define PROF_ARTIFICIAL_FRAMES 3
157ba2539a9Schs #endif
158ba2539a9Schs
159ba2539a9Schs #ifdef __aarch64__
160ba2539a9Schs /* TODO: verify */
161ba2539a9Schs #define PROF_ARTIFICIAL_FRAMES 10
162ba2539a9Schs #endif
163ba2539a9Schs
164ba2539a9Schs #ifdef __riscv__
165ba2539a9Schs /* TODO: verify */
166ba2539a9Schs #define PROF_ARTIFICIAL_FRAMES 10
167ba2539a9Schs #endif
168ba2539a9Schs
169ba2539a9Schs #endif /* __FreeBSD__ */
170ba2539a9Schs
1718f4ea3c4Schs #ifdef __NetBSD__
172*4c6ec8e8Sad #define PROF_ARTIFICIAL_FRAMES 4
1738f4ea3c4Schs #endif
174bb8023b5Sdarran
175bb8023b5Sdarran typedef struct profile_probe {
176bb8023b5Sdarran char prof_name[PROF_NAMELEN];
177bb8023b5Sdarran dtrace_id_t prof_id;
178bb8023b5Sdarran int prof_kind;
179ba2539a9Schs #if defined(illumos) || defined(__NetBSD__)
180bb8023b5Sdarran hrtime_t prof_interval;
181bb8023b5Sdarran cyclic_id_t prof_cyclic;
182ba2539a9Schs #endif
183ba2539a9Schs #ifdef __FreeBSD__
184ba2539a9Schs sbintime_t prof_interval;
185ba2539a9Schs struct callout prof_cyclic;
186ba2539a9Schs sbintime_t prof_expected;
187ba2539a9Schs struct profile_probe_percpu **prof_pcpus;
188ba2539a9Schs #endif
189bb8023b5Sdarran } profile_probe_t;
190bb8023b5Sdarran
191bb8023b5Sdarran typedef struct profile_probe_percpu {
192bb8023b5Sdarran hrtime_t profc_expected;
193bb8023b5Sdarran hrtime_t profc_interval;
194bb8023b5Sdarran profile_probe_t *profc_probe;
195ba2539a9Schs #ifdef __FreeBSD__
196ba2539a9Schs struct callout profc_cyclic;
197ba2539a9Schs #endif
198bb8023b5Sdarran } profile_probe_percpu_t;
199bb8023b5Sdarran
2008f4ea3c4Schs #ifdef __FreeBSD__
201bb8023b5Sdarran static d_open_t profile_open;
2028f4ea3c4Schs #endif
203bb8023b5Sdarran static int profile_unload(void);
204bb8023b5Sdarran static void profile_create(hrtime_t, char *, int);
205bb8023b5Sdarran static void profile_destroy(void *, dtrace_id_t, void *);
2068f4ea3c4Schs static int profile_enable(void *, dtrace_id_t, void *);
207bb8023b5Sdarran static void profile_disable(void *, dtrace_id_t, void *);
208bb8023b5Sdarran static void profile_load(void *);
209ba2539a9Schs static void profile_provide(void *, dtrace_probedesc_t *);
210bb8023b5Sdarran
211bb8023b5Sdarran static int profile_rates[] = {
212bb8023b5Sdarran 97, 199, 499, 997, 1999,
213bb8023b5Sdarran 4001, 4999, 0, 0, 0,
214bb8023b5Sdarran 0, 0, 0, 0, 0,
215bb8023b5Sdarran 0, 0, 0, 0, 0
216bb8023b5Sdarran };
217bb8023b5Sdarran
218bb8023b5Sdarran static int profile_ticks[] = {
219bb8023b5Sdarran 1, 10, 100, 500, 1000,
220bb8023b5Sdarran 5000, 0, 0, 0, 0,
221bb8023b5Sdarran 0, 0, 0, 0, 0
222bb8023b5Sdarran };
223bb8023b5Sdarran
224bb8023b5Sdarran /*
225bb8023b5Sdarran * profile_max defines the upper bound on the number of profile probes that
226bb8023b5Sdarran * can exist (this is to prevent malicious or clumsy users from exhausing
227bb8023b5Sdarran * system resources by creating a slew of profile probes). At mod load time,
228bb8023b5Sdarran * this gets its value from PROFILE_MAX_DEFAULT or profile-max-probes if it's
229bb8023b5Sdarran * present in the profile.conf file.
230bb8023b5Sdarran */
231bb8023b5Sdarran #define PROFILE_MAX_DEFAULT 1000 /* default max. number of probes */
232bb8023b5Sdarran static uint32_t profile_max = PROFILE_MAX_DEFAULT;
233bb8023b5Sdarran /* maximum number of profile probes */
234bb8023b5Sdarran static uint32_t profile_total; /* current number of profile probes */
235bb8023b5Sdarran
2368f4ea3c4Schs #ifdef __FreeBSD__
237bb8023b5Sdarran static struct cdevsw profile_cdevsw = {
238bb8023b5Sdarran .d_version = D_VERSION,
239bb8023b5Sdarran .d_open = profile_open,
240bb8023b5Sdarran .d_name = "profile",
241bb8023b5Sdarran };
2428f4ea3c4Schs #endif
243bb8023b5Sdarran
244bb8023b5Sdarran static dtrace_pattr_t profile_attr = {
245bb8023b5Sdarran { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
246bb8023b5Sdarran { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
247bb8023b5Sdarran { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
248bb8023b5Sdarran { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
249bb8023b5Sdarran { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
250bb8023b5Sdarran };
251bb8023b5Sdarran
252bb8023b5Sdarran static dtrace_pops_t profile_pops = {
253bb8023b5Sdarran profile_provide,
254bb8023b5Sdarran NULL,
255bb8023b5Sdarran profile_enable,
256bb8023b5Sdarran profile_disable,
257bb8023b5Sdarran NULL,
258bb8023b5Sdarran NULL,
259bb8023b5Sdarran NULL,
260bb8023b5Sdarran NULL,
261bb8023b5Sdarran NULL,
262bb8023b5Sdarran profile_destroy
263bb8023b5Sdarran };
264bb8023b5Sdarran
2658f4ea3c4Schs #ifdef __FreeBSD__
266bb8023b5Sdarran static struct cdev *profile_cdev;
2678f4ea3c4Schs #endif
268bb8023b5Sdarran static dtrace_provider_id_t profile_id;
269bb8023b5Sdarran static hrtime_t profile_interval_min = NANOSEC / 5000; /* 5000 hz */
270ba2539a9Schs static int profile_aframes = PROF_ARTIFICIAL_FRAMES;
271bb8023b5Sdarran
272ba2539a9Schs #ifdef __FreeBSD__
273ba2539a9Schs SYSCTL_DECL(_kern_dtrace);
274ba2539a9Schs SYSCTL_NODE(_kern_dtrace, OID_AUTO, profile, CTLFLAG_RD, 0, "DTrace profile parameters");
275ba2539a9Schs SYSCTL_INT(_kern_dtrace_profile, OID_AUTO, aframes, CTLFLAG_RW, &profile_aframes,
276ba2539a9Schs 0, "Skipped frames for profile provider");
277ba2539a9Schs
278ba2539a9Schs static sbintime_t
nsec_to_sbt(hrtime_t nsec)279ba2539a9Schs nsec_to_sbt(hrtime_t nsec)
280ba2539a9Schs {
281ba2539a9Schs time_t sec;
282ba2539a9Schs
283ba2539a9Schs /*
284ba2539a9Schs * We need to calculate nsec * 2^32 / 10^9
285ba2539a9Schs * Seconds and nanoseconds are split to avoid overflow.
286ba2539a9Schs */
287ba2539a9Schs sec = nsec / NANOSEC;
288ba2539a9Schs nsec = nsec % NANOSEC;
289ba2539a9Schs return (((sbintime_t)sec << 32) | ((sbintime_t)nsec << 32) / NANOSEC);
290ba2539a9Schs }
291ba2539a9Schs
292ba2539a9Schs static hrtime_t
sbt_to_nsec(sbintime_t sbt)293ba2539a9Schs sbt_to_nsec(sbintime_t sbt)
294ba2539a9Schs {
295ba2539a9Schs
296ba2539a9Schs return ((sbt >> 32) * NANOSEC +
297ba2539a9Schs (((uint32_t)sbt * (hrtime_t)NANOSEC) >> 32));
298ba2539a9Schs }
299ba2539a9Schs
300ba2539a9Schs static void
profile_fire(void * arg)301ba2539a9Schs profile_fire(void *arg)
302ba2539a9Schs {
303ba2539a9Schs profile_probe_percpu_t *pcpu = arg;
304ba2539a9Schs profile_probe_t *prof = pcpu->profc_probe;
305ba2539a9Schs hrtime_t late;
306ba2539a9Schs struct trapframe *frame;
307ba2539a9Schs uintfptr_t pc, upc;
308ba2539a9Schs
309ba2539a9Schs #ifdef illumos
310ba2539a9Schs late = gethrtime() - pcpu->profc_expected;
311ba2539a9Schs #else
312ba2539a9Schs late = sbt_to_nsec(sbinuptime() - pcpu->profc_expected);
313ba2539a9Schs #endif
314ba2539a9Schs
315ba2539a9Schs pc = 0;
316ba2539a9Schs upc = 0;
317ba2539a9Schs
318ba2539a9Schs /*
319ba2539a9Schs * td_intr_frame can be unset if this is a catch up event
320ba2539a9Schs * after waking up from idle sleep.
321ba2539a9Schs * This can only happen on a CPU idle thread.
322ba2539a9Schs */
323ba2539a9Schs frame = curthread->td_intr_frame;
324ba2539a9Schs if (frame != NULL) {
325ba2539a9Schs if (TRAPF_USERMODE(frame))
326ba2539a9Schs upc = TRAPF_PC(frame);
327ba2539a9Schs else
328ba2539a9Schs pc = TRAPF_PC(frame);
329ba2539a9Schs }
330ba2539a9Schs dtrace_probe(prof->prof_id, pc, upc, late, 0, 0);
331ba2539a9Schs
332ba2539a9Schs pcpu->profc_expected += pcpu->profc_interval;
333ba2539a9Schs callout_schedule_sbt_curcpu(&pcpu->profc_cyclic,
334ba2539a9Schs pcpu->profc_expected, 0, C_DIRECT_EXEC | C_ABSOLUTE);
335ba2539a9Schs }
336ba2539a9Schs
337ba2539a9Schs static void
profile_tick(void * arg)338ba2539a9Schs profile_tick(void *arg)
339ba2539a9Schs {
340ba2539a9Schs profile_probe_t *prof = arg;
341ba2539a9Schs struct trapframe *frame;
342ba2539a9Schs uintfptr_t pc, upc;
343ba2539a9Schs
344ba2539a9Schs pc = 0;
345ba2539a9Schs upc = 0;
346ba2539a9Schs
347ba2539a9Schs /*
348ba2539a9Schs * td_intr_frame can be unset if this is a catch up event
349ba2539a9Schs * after waking up from idle sleep.
350ba2539a9Schs * This can only happen on a CPU idle thread.
351ba2539a9Schs */
352ba2539a9Schs frame = curthread->td_intr_frame;
353ba2539a9Schs if (frame != NULL) {
354ba2539a9Schs if (TRAPF_USERMODE(frame))
355ba2539a9Schs upc = TRAPF_PC(frame);
356ba2539a9Schs else
357ba2539a9Schs pc = TRAPF_PC(frame);
358ba2539a9Schs }
359ba2539a9Schs dtrace_probe(prof->prof_id, pc, upc, 0, 0, 0);
360ba2539a9Schs
361ba2539a9Schs prof->prof_expected += prof->prof_interval;
362ba2539a9Schs callout_schedule_sbt(&prof->prof_cyclic,
363ba2539a9Schs prof->prof_expected, 0, C_DIRECT_EXEC | C_ABSOLUTE);
364ba2539a9Schs }
365ba2539a9Schs
366ba2539a9Schs #endif
367ba2539a9Schs
368ba2539a9Schs #ifdef __NetBSD__
369bb8023b5Sdarran static void
profile_fire(void * arg)370bb8023b5Sdarran profile_fire(void *arg)
371bb8023b5Sdarran {
372bb8023b5Sdarran profile_probe_percpu_t *pcpu = arg;
373bb8023b5Sdarran profile_probe_t *prof = pcpu->profc_probe;
374bb8023b5Sdarran hrtime_t late;
3758f4ea3c4Schs solaris_cpu_t *c = &solaris_cpu[cpu_number()];
376bb8023b5Sdarran
377bb8023b5Sdarran late = gethrtime() - pcpu->profc_expected;
378bb8023b5Sdarran pcpu->profc_expected += pcpu->profc_interval;
379bb8023b5Sdarran
380bb8023b5Sdarran dtrace_probe(prof->prof_id, c->cpu_profile_pc,
381bb8023b5Sdarran c->cpu_profile_upc, late, 0, 0);
382bb8023b5Sdarran }
383bb8023b5Sdarran
384bb8023b5Sdarran static void
profile_tick(void * arg)385bb8023b5Sdarran profile_tick(void *arg)
386bb8023b5Sdarran {
387bb8023b5Sdarran profile_probe_t *prof = arg;
3888f4ea3c4Schs solaris_cpu_t *c = &solaris_cpu[cpu_number()];
389bb8023b5Sdarran
390bb8023b5Sdarran dtrace_probe(prof->prof_id, c->cpu_profile_pc,
391bb8023b5Sdarran c->cpu_profile_upc, 0, 0, 0);
392bb8023b5Sdarran }
393bb8023b5Sdarran
394ba2539a9Schs #endif
395ba2539a9Schs
396bb8023b5Sdarran static void
profile_create(hrtime_t interval,char * name,int kind)397bb8023b5Sdarran profile_create(hrtime_t interval, char *name, int kind)
398bb8023b5Sdarran {
399bb8023b5Sdarran profile_probe_t *prof;
400bb8023b5Sdarran
401bb8023b5Sdarran if (interval < profile_interval_min)
402bb8023b5Sdarran return;
403bb8023b5Sdarran
404bb8023b5Sdarran if (dtrace_probe_lookup(profile_id, NULL, NULL, name) != 0)
405bb8023b5Sdarran return;
406bb8023b5Sdarran
407bb8023b5Sdarran atomic_add_32(&profile_total, 1);
408bb8023b5Sdarran if (profile_total > profile_max) {
409bb8023b5Sdarran atomic_add_32(&profile_total, -1);
410bb8023b5Sdarran return;
411bb8023b5Sdarran }
412bb8023b5Sdarran
413bb8023b5Sdarran prof = kmem_zalloc(sizeof (profile_probe_t), KM_SLEEP);
414bb8023b5Sdarran (void) strcpy(prof->prof_name, name);
415ba2539a9Schs #ifdef __FreeBSD__
416ba2539a9Schs prof->prof_interval = nsec_to_sbt(interval);
417ba2539a9Schs callout_init(&prof->prof_cyclic, 1);
418ba2539a9Schs #else
419bb8023b5Sdarran prof->prof_interval = interval;
420bb8023b5Sdarran prof->prof_cyclic = CYCLIC_NONE;
421ba2539a9Schs #endif
422bb8023b5Sdarran prof->prof_kind = kind;
423bb8023b5Sdarran prof->prof_id = dtrace_probe_create(profile_id,
424bb8023b5Sdarran NULL, NULL, name,
425ba2539a9Schs profile_aframes, prof);
426bb8023b5Sdarran }
427bb8023b5Sdarran
428bb8023b5Sdarran /*ARGSUSED*/
429bb8023b5Sdarran static void
profile_provide(void * arg,dtrace_probedesc_t * desc)430ba2539a9Schs profile_provide(void *arg, dtrace_probedesc_t *desc)
431bb8023b5Sdarran {
432bb8023b5Sdarran int i, j, rate, kind;
433bb8023b5Sdarran hrtime_t val = 0, mult = 1, len = 0;
434bb8023b5Sdarran char *name, *suffix = NULL;
435bb8023b5Sdarran
436bb8023b5Sdarran const struct {
437cd35dc3dSkamil char *prefix;
438bb8023b5Sdarran int kind;
439bb8023b5Sdarran } types[] = {
440bb8023b5Sdarran { PROF_PREFIX_PROFILE, PROF_PROFILE },
441bb8023b5Sdarran { PROF_PREFIX_TICK, PROF_TICK },
442bb8023b5Sdarran { 0, 0 }
443bb8023b5Sdarran };
444bb8023b5Sdarran
445bb8023b5Sdarran const struct {
446cd35dc3dSkamil char *name;
447bb8023b5Sdarran hrtime_t mult;
448bb8023b5Sdarran } suffixes[] = {
449bb8023b5Sdarran { "ns", NANOSEC / NANOSEC },
450bb8023b5Sdarran { "nsec", NANOSEC / NANOSEC },
451bb8023b5Sdarran { "us", NANOSEC / MICROSEC },
452bb8023b5Sdarran { "usec", NANOSEC / MICROSEC },
453bb8023b5Sdarran { "ms", NANOSEC / MILLISEC },
454bb8023b5Sdarran { "msec", NANOSEC / MILLISEC },
455bb8023b5Sdarran { "s", NANOSEC / SEC },
456bb8023b5Sdarran { "sec", NANOSEC / SEC },
457bb8023b5Sdarran { "m", NANOSEC * (hrtime_t)60 },
458bb8023b5Sdarran { "min", NANOSEC * (hrtime_t)60 },
459bb8023b5Sdarran { "h", NANOSEC * (hrtime_t)(60 * 60) },
460bb8023b5Sdarran { "hour", NANOSEC * (hrtime_t)(60 * 60) },
461bb8023b5Sdarran { "d", NANOSEC * (hrtime_t)(24 * 60 * 60) },
462bb8023b5Sdarran { "day", NANOSEC * (hrtime_t)(24 * 60 * 60) },
463bb8023b5Sdarran { "hz", 0 },
4648f4ea3c4Schs { NULL, 0 }
465bb8023b5Sdarran };
466bb8023b5Sdarran
467bb8023b5Sdarran if (desc == NULL) {
468bb8023b5Sdarran char n[PROF_NAMELEN];
469bb8023b5Sdarran
470bb8023b5Sdarran /*
471bb8023b5Sdarran * If no description was provided, provide all of our probes.
472bb8023b5Sdarran */
473bb8023b5Sdarran for (i = 0; i < sizeof (profile_rates) / sizeof (int); i++) {
474bb8023b5Sdarran if ((rate = profile_rates[i]) == 0)
475bb8023b5Sdarran continue;
476bb8023b5Sdarran
477bb8023b5Sdarran (void) snprintf(n, PROF_NAMELEN, "%s%d",
478bb8023b5Sdarran PROF_PREFIX_PROFILE, rate);
479bb8023b5Sdarran profile_create(NANOSEC / rate, n, PROF_PROFILE);
480bb8023b5Sdarran }
481bb8023b5Sdarran
482bb8023b5Sdarran for (i = 0; i < sizeof (profile_ticks) / sizeof (int); i++) {
483bb8023b5Sdarran if ((rate = profile_ticks[i]) == 0)
484bb8023b5Sdarran continue;
485bb8023b5Sdarran
486bb8023b5Sdarran (void) snprintf(n, PROF_NAMELEN, "%s%d",
487bb8023b5Sdarran PROF_PREFIX_TICK, rate);
488bb8023b5Sdarran profile_create(NANOSEC / rate, n, PROF_TICK);
489bb8023b5Sdarran }
490bb8023b5Sdarran
491bb8023b5Sdarran return;
492bb8023b5Sdarran }
493bb8023b5Sdarran
494ba2539a9Schs name = desc->dtpd_name;
495bb8023b5Sdarran
496bb8023b5Sdarran for (i = 0; types[i].prefix != NULL; i++) {
497bb8023b5Sdarran len = strlen(types[i].prefix);
498bb8023b5Sdarran
499bb8023b5Sdarran if (strncmp(name, types[i].prefix, len) != 0)
500bb8023b5Sdarran continue;
501bb8023b5Sdarran break;
502bb8023b5Sdarran }
503bb8023b5Sdarran
504bb8023b5Sdarran if (types[i].prefix == NULL)
505bb8023b5Sdarran return;
506bb8023b5Sdarran
507bb8023b5Sdarran kind = types[i].kind;
508bb8023b5Sdarran j = strlen(name) - len;
509bb8023b5Sdarran
510bb8023b5Sdarran /*
511bb8023b5Sdarran * We need to start before any time suffix.
512bb8023b5Sdarran */
513bb8023b5Sdarran for (j = strlen(name); j >= len; j--) {
514bb8023b5Sdarran if (name[j] >= '0' && name[j] <= '9')
515bb8023b5Sdarran break;
516bb8023b5Sdarran suffix = &name[j];
517bb8023b5Sdarran }
518bb8023b5Sdarran
519bb8023b5Sdarran ASSERT(suffix != NULL);
520bb8023b5Sdarran
521bb8023b5Sdarran /*
522bb8023b5Sdarran * Now determine the numerical value present in the probe name.
523bb8023b5Sdarran */
524bb8023b5Sdarran for (; j >= len; j--) {
525bb8023b5Sdarran if (name[j] < '0' || name[j] > '9')
526bb8023b5Sdarran return;
527bb8023b5Sdarran
528bb8023b5Sdarran val += (name[j] - '0') * mult;
529bb8023b5Sdarran mult *= (hrtime_t)10;
530bb8023b5Sdarran }
531bb8023b5Sdarran
532bb8023b5Sdarran if (val == 0)
533bb8023b5Sdarran return;
534bb8023b5Sdarran
535bb8023b5Sdarran /*
536bb8023b5Sdarran * Look-up the suffix to determine the multiplier.
537bb8023b5Sdarran */
538bb8023b5Sdarran for (i = 0, mult = 0; suffixes[i].name != NULL; i++) {
539bb8023b5Sdarran if (strcasecmp(suffixes[i].name, suffix) == 0) {
540bb8023b5Sdarran mult = suffixes[i].mult;
541bb8023b5Sdarran break;
542bb8023b5Sdarran }
543bb8023b5Sdarran }
544bb8023b5Sdarran
545bb8023b5Sdarran if (suffixes[i].name == NULL && *suffix != '\0')
546bb8023b5Sdarran return;
547bb8023b5Sdarran
548bb8023b5Sdarran if (mult == 0) {
549bb8023b5Sdarran /*
550bb8023b5Sdarran * The default is frequency-per-second.
551bb8023b5Sdarran */
552bb8023b5Sdarran val = NANOSEC / val;
553bb8023b5Sdarran } else {
554bb8023b5Sdarran val *= mult;
555bb8023b5Sdarran }
556bb8023b5Sdarran
557bb8023b5Sdarran profile_create(val, name, kind);
558bb8023b5Sdarran }
559bb8023b5Sdarran
560bb8023b5Sdarran /* ARGSUSED */
561bb8023b5Sdarran static void
profile_destroy(void * arg,dtrace_id_t id,void * parg)562bb8023b5Sdarran profile_destroy(void *arg, dtrace_id_t id, void *parg)
563bb8023b5Sdarran {
564bb8023b5Sdarran profile_probe_t *prof = parg;
565bb8023b5Sdarran
566ba2539a9Schs #ifdef __FreeBSD__
567ba2539a9Schs ASSERT(!callout_active(&prof->prof_cyclic) && prof->prof_pcpus == NULL);
568ba2539a9Schs #else
569bb8023b5Sdarran ASSERT(prof->prof_cyclic == CYCLIC_NONE);
570ba2539a9Schs #endif
571bb8023b5Sdarran kmem_free(prof, sizeof (profile_probe_t));
572bb8023b5Sdarran
573bb8023b5Sdarran ASSERT(profile_total >= 1);
574bb8023b5Sdarran atomic_add_32(&profile_total, -1);
575bb8023b5Sdarran }
576bb8023b5Sdarran
577ba2539a9Schs #ifndef __FreeBSD__
578ba2539a9Schs
579bb8023b5Sdarran /*ARGSUSED*/
580bb8023b5Sdarran static void
profile_online(void * arg,cpu_t * cpu,cyc_handler_t * hdlr,cyc_time_t * when)581bb8023b5Sdarran profile_online(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when)
582bb8023b5Sdarran {
583bb8023b5Sdarran profile_probe_t *prof = arg;
584bb8023b5Sdarran profile_probe_percpu_t *pcpu;
585bb8023b5Sdarran
586bb8023b5Sdarran pcpu = kmem_zalloc(sizeof (profile_probe_percpu_t), KM_SLEEP);
587bb8023b5Sdarran pcpu->profc_probe = prof;
588bb8023b5Sdarran
589bb8023b5Sdarran hdlr->cyh_func = profile_fire;
590bb8023b5Sdarran hdlr->cyh_arg = pcpu;
591bb8023b5Sdarran
592bb8023b5Sdarran when->cyt_interval = prof->prof_interval;
593bb8023b5Sdarran when->cyt_when = gethrtime() + when->cyt_interval;
594bb8023b5Sdarran
595bb8023b5Sdarran pcpu->profc_expected = when->cyt_when;
596bb8023b5Sdarran pcpu->profc_interval = when->cyt_interval;
597bb8023b5Sdarran }
598bb8023b5Sdarran
599bb8023b5Sdarran /*ARGSUSED*/
600bb8023b5Sdarran static void
profile_offline(void * arg,cpu_t * cpu,void * oarg)601bb8023b5Sdarran profile_offline(void *arg, cpu_t *cpu, void *oarg)
602bb8023b5Sdarran {
603bb8023b5Sdarran profile_probe_percpu_t *pcpu = oarg;
604bb8023b5Sdarran
605bb8023b5Sdarran ASSERT(pcpu->profc_probe == arg);
606bb8023b5Sdarran kmem_free(pcpu, sizeof (profile_probe_percpu_t));
607bb8023b5Sdarran }
608bb8023b5Sdarran
609bb8023b5Sdarran /* ARGSUSED */
6108f4ea3c4Schs static int
profile_enable(void * arg,dtrace_id_t id,void * parg)611bb8023b5Sdarran profile_enable(void *arg, dtrace_id_t id, void *parg)
612bb8023b5Sdarran {
613bb8023b5Sdarran profile_probe_t *prof = parg;
614bb8023b5Sdarran cyc_omni_handler_t omni;
615bb8023b5Sdarran cyc_handler_t hdlr;
616bb8023b5Sdarran cyc_time_t when;
617bb8023b5Sdarran
618bb8023b5Sdarran ASSERT(prof->prof_interval != 0);
619bb8023b5Sdarran ASSERT(MUTEX_HELD(&cpu_lock));
620bb8023b5Sdarran
621bb8023b5Sdarran if (prof->prof_kind == PROF_TICK) {
622bb8023b5Sdarran hdlr.cyh_func = profile_tick;
623bb8023b5Sdarran hdlr.cyh_arg = prof;
624bb8023b5Sdarran
625bb8023b5Sdarran when.cyt_interval = prof->prof_interval;
626bb8023b5Sdarran when.cyt_when = gethrtime() + when.cyt_interval;
627bb8023b5Sdarran } else {
628bb8023b5Sdarran ASSERT(prof->prof_kind == PROF_PROFILE);
629bb8023b5Sdarran omni.cyo_online = profile_online;
630bb8023b5Sdarran omni.cyo_offline = profile_offline;
631bb8023b5Sdarran omni.cyo_arg = prof;
632bb8023b5Sdarran }
633bb8023b5Sdarran
634bb8023b5Sdarran if (prof->prof_kind == PROF_TICK) {
635bb8023b5Sdarran prof->prof_cyclic = cyclic_add(&hdlr, &when);
636bb8023b5Sdarran } else {
637bb8023b5Sdarran prof->prof_cyclic = cyclic_add_omni(&omni);
638bb8023b5Sdarran }
6398f4ea3c4Schs return 0;
640bb8023b5Sdarran }
641bb8023b5Sdarran
642bb8023b5Sdarran /* ARGSUSED */
643bb8023b5Sdarran static void
profile_disable(void * arg,dtrace_id_t id,void * parg)644bb8023b5Sdarran profile_disable(void *arg, dtrace_id_t id, void *parg)
645bb8023b5Sdarran {
646bb8023b5Sdarran profile_probe_t *prof = parg;
647bb8023b5Sdarran
648bb8023b5Sdarran ASSERT(prof->prof_cyclic != CYCLIC_NONE);
649bb8023b5Sdarran ASSERT(MUTEX_HELD(&cpu_lock));
650bb8023b5Sdarran
651bb8023b5Sdarran cyclic_remove(prof->prof_cyclic);
652bb8023b5Sdarran prof->prof_cyclic = CYCLIC_NONE;
653bb8023b5Sdarran }
654bb8023b5Sdarran
655ba2539a9Schs #else
656ba2539a9Schs
657ba2539a9Schs static void
profile_enable_omni(profile_probe_t * prof)658ba2539a9Schs profile_enable_omni(profile_probe_t *prof)
659ba2539a9Schs {
660ba2539a9Schs profile_probe_percpu_t *pcpu;
661ba2539a9Schs int cpu;
662ba2539a9Schs
663ba2539a9Schs prof->prof_pcpus = kmem_zalloc((mp_maxid + 1) * sizeof(pcpu), KM_SLEEP);
664ba2539a9Schs CPU_FOREACH(cpu) {
665ba2539a9Schs pcpu = kmem_zalloc(sizeof(profile_probe_percpu_t), KM_SLEEP);
666ba2539a9Schs prof->prof_pcpus[cpu] = pcpu;
667ba2539a9Schs pcpu->profc_probe = prof;
668ba2539a9Schs pcpu->profc_expected = sbinuptime() + prof->prof_interval;
669ba2539a9Schs pcpu->profc_interval = prof->prof_interval;
670ba2539a9Schs callout_init(&pcpu->profc_cyclic, 1);
671ba2539a9Schs callout_reset_sbt_on(&pcpu->profc_cyclic,
672ba2539a9Schs pcpu->profc_expected, 0, profile_fire, pcpu,
673ba2539a9Schs cpu, C_DIRECT_EXEC | C_ABSOLUTE);
674ba2539a9Schs }
675ba2539a9Schs }
676ba2539a9Schs
677ba2539a9Schs static void
profile_disable_omni(profile_probe_t * prof)678ba2539a9Schs profile_disable_omni(profile_probe_t *prof)
679ba2539a9Schs {
680ba2539a9Schs profile_probe_percpu_t *pcpu;
681ba2539a9Schs int cpu;
682ba2539a9Schs
683ba2539a9Schs ASSERT(prof->prof_pcpus != NULL);
684ba2539a9Schs CPU_FOREACH(cpu) {
685ba2539a9Schs pcpu = prof->prof_pcpus[cpu];
686ba2539a9Schs ASSERT(pcpu->profc_probe == prof);
687ba2539a9Schs ASSERT(callout_active(&pcpu->profc_cyclic));
688ba2539a9Schs callout_stop(&pcpu->profc_cyclic);
689ba2539a9Schs callout_drain(&pcpu->profc_cyclic);
690ba2539a9Schs kmem_free(pcpu, sizeof(profile_probe_percpu_t));
691ba2539a9Schs }
692ba2539a9Schs kmem_free(prof->prof_pcpus, (mp_maxid + 1) * sizeof(pcpu));
693ba2539a9Schs prof->prof_pcpus = NULL;
694ba2539a9Schs }
695ba2539a9Schs
696ba2539a9Schs /* ARGSUSED */
697ba2539a9Schs static void
profile_enable(void * arg,dtrace_id_t id,void * parg)698ba2539a9Schs profile_enable(void *arg, dtrace_id_t id, void *parg)
699ba2539a9Schs {
700ba2539a9Schs profile_probe_t *prof = parg;
701ba2539a9Schs
702ba2539a9Schs if (prof->prof_kind == PROF_TICK) {
703ba2539a9Schs prof->prof_expected = sbinuptime() + prof->prof_interval;
704ba2539a9Schs callout_reset_sbt(&prof->prof_cyclic,
705ba2539a9Schs prof->prof_expected, 0, profile_tick, prof,
706ba2539a9Schs C_DIRECT_EXEC | C_ABSOLUTE);
707ba2539a9Schs } else {
708ba2539a9Schs ASSERT(prof->prof_kind == PROF_PROFILE);
709ba2539a9Schs profile_enable_omni(prof);
710ba2539a9Schs }
711ba2539a9Schs }
712ba2539a9Schs
713ba2539a9Schs /* ARGSUSED */
714ba2539a9Schs static void
profile_disable(void * arg,dtrace_id_t id,void * parg)715ba2539a9Schs profile_disable(void *arg, dtrace_id_t id, void *parg)
716ba2539a9Schs {
717ba2539a9Schs profile_probe_t *prof = parg;
718ba2539a9Schs
719ba2539a9Schs if (prof->prof_kind == PROF_TICK) {
720ba2539a9Schs ASSERT(callout_active(&prof->prof_cyclic));
721ba2539a9Schs callout_stop(&prof->prof_cyclic);
722ba2539a9Schs callout_drain(&prof->prof_cyclic);
723ba2539a9Schs } else {
724ba2539a9Schs ASSERT(prof->prof_kind == PROF_PROFILE);
725ba2539a9Schs profile_disable_omni(prof);
726ba2539a9Schs }
727ba2539a9Schs }
728ba2539a9Schs #endif
729ba2539a9Schs
730bb8023b5Sdarran static void
profile_load(void * dummy)731bb8023b5Sdarran profile_load(void *dummy)
732bb8023b5Sdarran {
7338f4ea3c4Schs #ifdef __FreeBSD__
734bb8023b5Sdarran /* Create the /dev/dtrace/profile entry. */
735bb8023b5Sdarran profile_cdev = make_dev(&profile_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
736bb8023b5Sdarran "dtrace/profile");
7378f4ea3c4Schs #endif
738bb8023b5Sdarran
739bb8023b5Sdarran if (dtrace_register("profile", &profile_attr, DTRACE_PRIV_USER,
740bb8023b5Sdarran NULL, &profile_pops, NULL, &profile_id) != 0)
741bb8023b5Sdarran return;
742bb8023b5Sdarran }
743bb8023b5Sdarran
744bb8023b5Sdarran
745bb8023b5Sdarran static int
profile_unload()746bb8023b5Sdarran profile_unload()
747bb8023b5Sdarran {
748bb8023b5Sdarran int error = 0;
749bb8023b5Sdarran
750bb8023b5Sdarran if ((error = dtrace_unregister(profile_id)) != 0)
751bb8023b5Sdarran return (error);
752bb8023b5Sdarran
7538f4ea3c4Schs #ifdef __FreeBSD__
754bb8023b5Sdarran destroy_dev(profile_cdev);
7558f4ea3c4Schs #endif
756bb8023b5Sdarran
757bb8023b5Sdarran return (error);
758bb8023b5Sdarran }
759bb8023b5Sdarran
7608f4ea3c4Schs #ifdef __FreeBSD__
7618f4ea3c4Schs
762bb8023b5Sdarran /* ARGSUSED */
763bb8023b5Sdarran static int
profile_modevent(module_t mod __unused,int type,void * data __unused)764bb8023b5Sdarran profile_modevent(module_t mod __unused, int type, void *data __unused)
765bb8023b5Sdarran {
766bb8023b5Sdarran int error = 0;
767bb8023b5Sdarran
768bb8023b5Sdarran switch (type) {
769bb8023b5Sdarran case MOD_LOAD:
770bb8023b5Sdarran break;
771bb8023b5Sdarran
772bb8023b5Sdarran case MOD_UNLOAD:
773bb8023b5Sdarran break;
774bb8023b5Sdarran
775bb8023b5Sdarran case MOD_SHUTDOWN:
776bb8023b5Sdarran break;
777bb8023b5Sdarran
778bb8023b5Sdarran default:
779bb8023b5Sdarran error = EOPNOTSUPP;
780bb8023b5Sdarran break;
781bb8023b5Sdarran
782bb8023b5Sdarran }
783bb8023b5Sdarran return (error);
784bb8023b5Sdarran }
785bb8023b5Sdarran
786bb8023b5Sdarran /* ARGSUSED */
787bb8023b5Sdarran static int
profile_open(struct cdev * dev __unused,int oflags __unused,int devtype __unused,struct thread * td __unused)788bb8023b5Sdarran profile_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused, struct thread *td __unused)
789bb8023b5Sdarran {
790bb8023b5Sdarran return (0);
791bb8023b5Sdarran }
792bb8023b5Sdarran
793bb8023b5Sdarran SYSINIT(profile_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, profile_load, NULL);
794bb8023b5Sdarran SYSUNINIT(profile_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, profile_unload, NULL);
795bb8023b5Sdarran
796bb8023b5Sdarran DEV_MODULE(profile, profile_modevent, NULL);
797bb8023b5Sdarran MODULE_VERSION(profile, 1);
798bb8023b5Sdarran MODULE_DEPEND(profile, dtrace, 1, 1, 1);
799bb8023b5Sdarran MODULE_DEPEND(profile, cyclic, 1, 1, 1);
800bb8023b5Sdarran MODULE_DEPEND(profile, opensolaris, 1, 1, 1);
8018f4ea3c4Schs
8028f4ea3c4Schs #endif
8038f4ea3c4Schs
8048f4ea3c4Schs #ifdef __NetBSD__
8058f4ea3c4Schs
8068f4ea3c4Schs static int
dtrace_profile_modcmd(modcmd_t cmd,void * data)8070def3bc2Sriastradh dtrace_profile_modcmd(modcmd_t cmd, void *data)
8088f4ea3c4Schs {
8098f4ea3c4Schs switch (cmd) {
8108f4ea3c4Schs case MODULE_CMD_INIT:
8118f4ea3c4Schs profile_load(NULL);
8128f4ea3c4Schs return 0;
8138f4ea3c4Schs
8148f4ea3c4Schs case MODULE_CMD_FINI:
8158f4ea3c4Schs profile_unload();
8168f4ea3c4Schs return 0;
8178f4ea3c4Schs
81851346d28Sriastradh case MODULE_CMD_AUTOUNLOAD:
81951346d28Sriastradh if (profile_total)
82051346d28Sriastradh return EBUSY;
82151346d28Sriastradh return 0;
82251346d28Sriastradh
8238f4ea3c4Schs default:
8248f4ea3c4Schs return ENOTTY;
8258f4ea3c4Schs }
8268f4ea3c4Schs }
8278f4ea3c4Schs
8280def3bc2Sriastradh MODULE(MODULE_CLASS_MISC, dtrace_profile, "dtrace,cyclic");
8298f4ea3c4Schs
8308f4ea3c4Schs #endif
831