10Sstevel@tonic-gate /*
20Sstevel@tonic-gate * CDDL HEADER START
30Sstevel@tonic-gate *
40Sstevel@tonic-gate * The contents of this file are subject to the terms of the
51880Sahl * Common Development and Distribution License (the "License").
61880Sahl * You may not use this file except in compliance with the License.
70Sstevel@tonic-gate *
80Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
90Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
100Sstevel@tonic-gate * See the License for the specific language governing permissions
110Sstevel@tonic-gate * and limitations under the License.
120Sstevel@tonic-gate *
130Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
140Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
150Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
160Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
170Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
180Sstevel@tonic-gate *
190Sstevel@tonic-gate * CDDL HEADER END
200Sstevel@tonic-gate */
211880Sahl
220Sstevel@tonic-gate /*
23*8803SJonathan.Haslam@Sun.COM * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
240Sstevel@tonic-gate * Use is subject to license terms.
250Sstevel@tonic-gate */
260Sstevel@tonic-gate
270Sstevel@tonic-gate #include <sys/dtrace.h>
280Sstevel@tonic-gate #include <sys/cmn_err.h>
290Sstevel@tonic-gate #include <sys/tnf.h>
300Sstevel@tonic-gate #include <sys/atomic.h>
310Sstevel@tonic-gate #include <sys/prsystm.h>
320Sstevel@tonic-gate #include <sys/modctl.h>
330Sstevel@tonic-gate #include <sys/aio_impl.h>
340Sstevel@tonic-gate
350Sstevel@tonic-gate #ifdef __sparc
360Sstevel@tonic-gate #include <sys/privregs.h>
370Sstevel@tonic-gate #endif
380Sstevel@tonic-gate
390Sstevel@tonic-gate void (*dtrace_cpu_init)(processorid_t);
400Sstevel@tonic-gate void (*dtrace_modload)(struct modctl *);
410Sstevel@tonic-gate void (*dtrace_modunload)(struct modctl *);
420Sstevel@tonic-gate void (*dtrace_helpers_cleanup)(void);
430Sstevel@tonic-gate void (*dtrace_helpers_fork)(proc_t *, proc_t *);
440Sstevel@tonic-gate void (*dtrace_cpustart_init)(void);
450Sstevel@tonic-gate void (*dtrace_cpustart_fini)(void);
46*8803SJonathan.Haslam@Sun.COM void (*dtrace_cpc_fire)(uint64_t);
470Sstevel@tonic-gate
480Sstevel@tonic-gate void (*dtrace_debugger_init)(void);
490Sstevel@tonic-gate void (*dtrace_debugger_fini)(void);
500Sstevel@tonic-gate
510Sstevel@tonic-gate dtrace_vtime_state_t dtrace_vtime_active = 0;
520Sstevel@tonic-gate dtrace_cacheid_t dtrace_predcache_id = DTRACE_CACHEIDNONE + 1;
530Sstevel@tonic-gate
54*8803SJonathan.Haslam@Sun.COM /*
55*8803SJonathan.Haslam@Sun.COM * dtrace_cpc_in_use usage statement: this global variable is used by the cpc
56*8803SJonathan.Haslam@Sun.COM * hardware overflow interrupt handler and the kernel cpc framework to check
57*8803SJonathan.Haslam@Sun.COM * whether or not the DTrace cpc provider is currently in use. The variable is
58*8803SJonathan.Haslam@Sun.COM * set before counters are enabled with the first enabling and cleared when
59*8803SJonathan.Haslam@Sun.COM * the last enabling is disabled. Its value at any given time indicates the
60*8803SJonathan.Haslam@Sun.COM * number of active dcpc based enablings. The global 'kcpc_cpuctx_lock' rwlock
61*8803SJonathan.Haslam@Sun.COM * is held during initial setting to protect races between kcpc_open() and the
62*8803SJonathan.Haslam@Sun.COM * first enabling. The locking provided by the DTrace subsystem, the kernel
63*8803SJonathan.Haslam@Sun.COM * cpc framework and the cpu management framework protect consumers from race
64*8803SJonathan.Haslam@Sun.COM * conditions on enabling and disabling probes.
65*8803SJonathan.Haslam@Sun.COM */
66*8803SJonathan.Haslam@Sun.COM uint32_t dtrace_cpc_in_use = 0;
67*8803SJonathan.Haslam@Sun.COM
680Sstevel@tonic-gate typedef struct dtrace_hrestime {
690Sstevel@tonic-gate lock_t dthr_lock; /* lock for this element */
700Sstevel@tonic-gate timestruc_t dthr_hrestime; /* hrestime value */
710Sstevel@tonic-gate int64_t dthr_adj; /* hrestime_adj value */
720Sstevel@tonic-gate hrtime_t dthr_hrtime; /* hrtime value */
730Sstevel@tonic-gate } dtrace_hrestime_t;
740Sstevel@tonic-gate
750Sstevel@tonic-gate static dtrace_hrestime_t dtrace_hrestime[2];
760Sstevel@tonic-gate
770Sstevel@tonic-gate /*
780Sstevel@tonic-gate * Making available adjustable high-resolution time in DTrace is regrettably
790Sstevel@tonic-gate * more complicated than one might think it should be. The problem is that
800Sstevel@tonic-gate * the variables related to adjusted high-resolution time (hrestime,
810Sstevel@tonic-gate * hrestime_adj and friends) are adjusted under hres_lock -- and this lock may
820Sstevel@tonic-gate * be held when we enter probe context. One might think that we could address
830Sstevel@tonic-gate * this by having a single snapshot copy that is stored under a different lock
840Sstevel@tonic-gate * from hres_tick(), using the snapshot iff hres_lock is locked in probe
850Sstevel@tonic-gate * context. Unfortunately, this too won't work: because hres_lock is grabbed
860Sstevel@tonic-gate * in more than just hres_tick() context, we could enter probe context
870Sstevel@tonic-gate * concurrently on two different CPUs with both locks (hres_lock and the
880Sstevel@tonic-gate * snapshot lock) held. As this implies, the fundamental problem is that we
890Sstevel@tonic-gate * need to have access to a snapshot of these variables that we _know_ will
900Sstevel@tonic-gate * not be locked in probe context. To effect this, we have two snapshots
910Sstevel@tonic-gate * protected by two different locks, and we mandate that these snapshots are
920Sstevel@tonic-gate * recorded in succession by a single thread calling dtrace_hres_tick(). (We
930Sstevel@tonic-gate * assure this by calling it out of the same CY_HIGH_LEVEL cyclic that calls
940Sstevel@tonic-gate * hres_tick().) A single thread can't be in two places at once: one of the
950Sstevel@tonic-gate * snapshot locks is guaranteed to be unheld at all times. The
960Sstevel@tonic-gate * dtrace_gethrestime() algorithm is thus to check first one snapshot and then
970Sstevel@tonic-gate * the other to find the unlocked snapshot.
980Sstevel@tonic-gate */
990Sstevel@tonic-gate void
dtrace_hres_tick(void)1000Sstevel@tonic-gate dtrace_hres_tick(void)
1010Sstevel@tonic-gate {
1020Sstevel@tonic-gate int i;
1030Sstevel@tonic-gate ushort_t spl;
1040Sstevel@tonic-gate
1050Sstevel@tonic-gate for (i = 0; i < 2; i++) {
1060Sstevel@tonic-gate dtrace_hrestime_t tmp;
1070Sstevel@tonic-gate
1080Sstevel@tonic-gate spl = hr_clock_lock();
1090Sstevel@tonic-gate tmp.dthr_hrestime = hrestime;
1100Sstevel@tonic-gate tmp.dthr_adj = hrestime_adj;
1110Sstevel@tonic-gate tmp.dthr_hrtime = dtrace_gethrtime();
1120Sstevel@tonic-gate hr_clock_unlock(spl);
1130Sstevel@tonic-gate
1140Sstevel@tonic-gate lock_set(&dtrace_hrestime[i].dthr_lock);
1150Sstevel@tonic-gate dtrace_hrestime[i].dthr_hrestime = tmp.dthr_hrestime;
1160Sstevel@tonic-gate dtrace_hrestime[i].dthr_adj = tmp.dthr_adj;
1170Sstevel@tonic-gate dtrace_hrestime[i].dthr_hrtime = tmp.dthr_hrtime;
1180Sstevel@tonic-gate dtrace_membar_producer();
1190Sstevel@tonic-gate
1200Sstevel@tonic-gate /*
1210Sstevel@tonic-gate * To allow for lock-free examination of this lock, we use
1220Sstevel@tonic-gate * the same trick that is used hres_lock; for more details,
1230Sstevel@tonic-gate * see the description of this technique in sun4u/sys/clock.h.
1240Sstevel@tonic-gate */
1250Sstevel@tonic-gate dtrace_hrestime[i].dthr_lock++;
1260Sstevel@tonic-gate }
1270Sstevel@tonic-gate }
1280Sstevel@tonic-gate
1290Sstevel@tonic-gate hrtime_t
dtrace_gethrestime(void)1300Sstevel@tonic-gate dtrace_gethrestime(void)
1310Sstevel@tonic-gate {
1320Sstevel@tonic-gate dtrace_hrestime_t snap;
1330Sstevel@tonic-gate hrtime_t now;
1340Sstevel@tonic-gate int i = 0, adj, nslt;
1350Sstevel@tonic-gate
1360Sstevel@tonic-gate for (;;) {
1370Sstevel@tonic-gate snap.dthr_lock = dtrace_hrestime[i].dthr_lock;
1380Sstevel@tonic-gate dtrace_membar_consumer();
1390Sstevel@tonic-gate snap.dthr_hrestime = dtrace_hrestime[i].dthr_hrestime;
1400Sstevel@tonic-gate snap.dthr_hrtime = dtrace_hrestime[i].dthr_hrtime;
1410Sstevel@tonic-gate snap.dthr_adj = dtrace_hrestime[i].dthr_adj;
1420Sstevel@tonic-gate dtrace_membar_consumer();
1430Sstevel@tonic-gate
1440Sstevel@tonic-gate if ((snap.dthr_lock & ~1) == dtrace_hrestime[i].dthr_lock)
1450Sstevel@tonic-gate break;
1460Sstevel@tonic-gate
1470Sstevel@tonic-gate /*
1480Sstevel@tonic-gate * If we're here, the lock was either locked, or it
1490Sstevel@tonic-gate * transitioned while we were taking the snapshot. Either
1500Sstevel@tonic-gate * way, we're going to try the other dtrace_hrestime element;
1510Sstevel@tonic-gate * we know that it isn't possible for both to be locked
1520Sstevel@tonic-gate * simultaneously, so we will ultimately get a good snapshot.
1530Sstevel@tonic-gate */
1540Sstevel@tonic-gate i ^= 1;
1550Sstevel@tonic-gate }
1560Sstevel@tonic-gate
1570Sstevel@tonic-gate /*
1580Sstevel@tonic-gate * We have a good snapshot. Now perform any necessary adjustments.
1590Sstevel@tonic-gate */
1600Sstevel@tonic-gate nslt = dtrace_gethrtime() - snap.dthr_hrtime;
1610Sstevel@tonic-gate ASSERT(nslt >= 0);
1620Sstevel@tonic-gate
1630Sstevel@tonic-gate now = ((hrtime_t)snap.dthr_hrestime.tv_sec * (hrtime_t)NANOSEC) +
1640Sstevel@tonic-gate snap.dthr_hrestime.tv_nsec;
1650Sstevel@tonic-gate
1660Sstevel@tonic-gate if (snap.dthr_adj != 0) {
1670Sstevel@tonic-gate if (snap.dthr_adj > 0) {
1680Sstevel@tonic-gate adj = (nslt >> adj_shift);
1690Sstevel@tonic-gate if (adj > snap.dthr_adj)
1700Sstevel@tonic-gate adj = (int)snap.dthr_adj;
1710Sstevel@tonic-gate } else {
1720Sstevel@tonic-gate adj = -(nslt >> adj_shift);
1730Sstevel@tonic-gate if (adj < snap.dthr_adj)
1740Sstevel@tonic-gate adj = (int)snap.dthr_adj;
1750Sstevel@tonic-gate }
1760Sstevel@tonic-gate now += adj;
1770Sstevel@tonic-gate }
1780Sstevel@tonic-gate
1790Sstevel@tonic-gate return (now);
1800Sstevel@tonic-gate }
1810Sstevel@tonic-gate
1820Sstevel@tonic-gate void
dtrace_vtime_enable(void)1830Sstevel@tonic-gate dtrace_vtime_enable(void)
1840Sstevel@tonic-gate {
1850Sstevel@tonic-gate dtrace_vtime_state_t state, nstate;
1860Sstevel@tonic-gate
1870Sstevel@tonic-gate do {
1880Sstevel@tonic-gate state = dtrace_vtime_active;
1890Sstevel@tonic-gate
1900Sstevel@tonic-gate switch (state) {
1910Sstevel@tonic-gate case DTRACE_VTIME_INACTIVE:
1920Sstevel@tonic-gate nstate = DTRACE_VTIME_ACTIVE;
1930Sstevel@tonic-gate break;
1940Sstevel@tonic-gate
1950Sstevel@tonic-gate case DTRACE_VTIME_INACTIVE_TNF:
1960Sstevel@tonic-gate nstate = DTRACE_VTIME_ACTIVE_TNF;
1970Sstevel@tonic-gate break;
1980Sstevel@tonic-gate
1990Sstevel@tonic-gate case DTRACE_VTIME_ACTIVE:
2000Sstevel@tonic-gate case DTRACE_VTIME_ACTIVE_TNF:
2010Sstevel@tonic-gate panic("DTrace virtual time already enabled");
2020Sstevel@tonic-gate /*NOTREACHED*/
2030Sstevel@tonic-gate }
2040Sstevel@tonic-gate
2050Sstevel@tonic-gate } while (cas32((uint32_t *)&dtrace_vtime_active,
2060Sstevel@tonic-gate state, nstate) != state);
2070Sstevel@tonic-gate }
2080Sstevel@tonic-gate
2090Sstevel@tonic-gate void
dtrace_vtime_disable(void)2100Sstevel@tonic-gate dtrace_vtime_disable(void)
2110Sstevel@tonic-gate {
2120Sstevel@tonic-gate dtrace_vtime_state_t state, nstate;
2130Sstevel@tonic-gate
2140Sstevel@tonic-gate do {
2150Sstevel@tonic-gate state = dtrace_vtime_active;
2160Sstevel@tonic-gate
2170Sstevel@tonic-gate switch (state) {
2180Sstevel@tonic-gate case DTRACE_VTIME_ACTIVE:
2190Sstevel@tonic-gate nstate = DTRACE_VTIME_INACTIVE;
2200Sstevel@tonic-gate break;
2210Sstevel@tonic-gate
2220Sstevel@tonic-gate case DTRACE_VTIME_ACTIVE_TNF:
2230Sstevel@tonic-gate nstate = DTRACE_VTIME_INACTIVE_TNF;
2240Sstevel@tonic-gate break;
2250Sstevel@tonic-gate
2260Sstevel@tonic-gate case DTRACE_VTIME_INACTIVE:
2270Sstevel@tonic-gate case DTRACE_VTIME_INACTIVE_TNF:
2280Sstevel@tonic-gate panic("DTrace virtual time already disabled");
2290Sstevel@tonic-gate /*NOTREACHED*/
2300Sstevel@tonic-gate }
2310Sstevel@tonic-gate
2320Sstevel@tonic-gate } while (cas32((uint32_t *)&dtrace_vtime_active,
2330Sstevel@tonic-gate state, nstate) != state);
2340Sstevel@tonic-gate }
2350Sstevel@tonic-gate
2360Sstevel@tonic-gate void
dtrace_vtime_enable_tnf(void)2370Sstevel@tonic-gate dtrace_vtime_enable_tnf(void)
2380Sstevel@tonic-gate {
2390Sstevel@tonic-gate dtrace_vtime_state_t state, nstate;
2400Sstevel@tonic-gate
2410Sstevel@tonic-gate do {
2420Sstevel@tonic-gate state = dtrace_vtime_active;
2430Sstevel@tonic-gate
2440Sstevel@tonic-gate switch (state) {
2450Sstevel@tonic-gate case DTRACE_VTIME_ACTIVE:
2460Sstevel@tonic-gate nstate = DTRACE_VTIME_ACTIVE_TNF;
2470Sstevel@tonic-gate break;
2480Sstevel@tonic-gate
2490Sstevel@tonic-gate case DTRACE_VTIME_INACTIVE:
2500Sstevel@tonic-gate nstate = DTRACE_VTIME_INACTIVE_TNF;
2510Sstevel@tonic-gate break;
2520Sstevel@tonic-gate
2530Sstevel@tonic-gate case DTRACE_VTIME_ACTIVE_TNF:
2540Sstevel@tonic-gate case DTRACE_VTIME_INACTIVE_TNF:
2550Sstevel@tonic-gate panic("TNF already active");
2560Sstevel@tonic-gate /*NOTREACHED*/
2570Sstevel@tonic-gate }
2580Sstevel@tonic-gate
2590Sstevel@tonic-gate } while (cas32((uint32_t *)&dtrace_vtime_active,
2600Sstevel@tonic-gate state, nstate) != state);
2610Sstevel@tonic-gate }
2620Sstevel@tonic-gate
2630Sstevel@tonic-gate void
dtrace_vtime_disable_tnf(void)2640Sstevel@tonic-gate dtrace_vtime_disable_tnf(void)
2650Sstevel@tonic-gate {
2660Sstevel@tonic-gate dtrace_vtime_state_t state, nstate;
2670Sstevel@tonic-gate
2680Sstevel@tonic-gate do {
2690Sstevel@tonic-gate state = dtrace_vtime_active;
2700Sstevel@tonic-gate
2710Sstevel@tonic-gate switch (state) {
2720Sstevel@tonic-gate case DTRACE_VTIME_ACTIVE_TNF:
2730Sstevel@tonic-gate nstate = DTRACE_VTIME_ACTIVE;
2740Sstevel@tonic-gate break;
2750Sstevel@tonic-gate
2760Sstevel@tonic-gate case DTRACE_VTIME_INACTIVE_TNF:
2770Sstevel@tonic-gate nstate = DTRACE_VTIME_INACTIVE;
2780Sstevel@tonic-gate break;
2790Sstevel@tonic-gate
2800Sstevel@tonic-gate case DTRACE_VTIME_ACTIVE:
2810Sstevel@tonic-gate case DTRACE_VTIME_INACTIVE:
2820Sstevel@tonic-gate panic("TNF already inactive");
2830Sstevel@tonic-gate /*NOTREACHED*/
2840Sstevel@tonic-gate }
2850Sstevel@tonic-gate
2860Sstevel@tonic-gate } while (cas32((uint32_t *)&dtrace_vtime_active,
2870Sstevel@tonic-gate state, nstate) != state);
2880Sstevel@tonic-gate }
2890Sstevel@tonic-gate
2900Sstevel@tonic-gate void
dtrace_vtime_switch(kthread_t * next)2910Sstevel@tonic-gate dtrace_vtime_switch(kthread_t *next)
2920Sstevel@tonic-gate {
2930Sstevel@tonic-gate dtrace_icookie_t cookie;
2940Sstevel@tonic-gate hrtime_t ts;
2950Sstevel@tonic-gate
2960Sstevel@tonic-gate if (tnf_tracing_active) {
2970Sstevel@tonic-gate tnf_thread_switch(next);
2980Sstevel@tonic-gate
2990Sstevel@tonic-gate if (dtrace_vtime_active == DTRACE_VTIME_INACTIVE_TNF)
3000Sstevel@tonic-gate return;
3010Sstevel@tonic-gate }
3020Sstevel@tonic-gate
3030Sstevel@tonic-gate cookie = dtrace_interrupt_disable();
3040Sstevel@tonic-gate ts = dtrace_gethrtime();
3050Sstevel@tonic-gate
3060Sstevel@tonic-gate if (curthread->t_dtrace_start != 0) {
3070Sstevel@tonic-gate curthread->t_dtrace_vtime += ts - curthread->t_dtrace_start;
3080Sstevel@tonic-gate curthread->t_dtrace_start = 0;
3090Sstevel@tonic-gate }
3100Sstevel@tonic-gate
3110Sstevel@tonic-gate next->t_dtrace_start = ts;
3120Sstevel@tonic-gate
3130Sstevel@tonic-gate dtrace_interrupt_enable(cookie);
3140Sstevel@tonic-gate }
3150Sstevel@tonic-gate
3160Sstevel@tonic-gate void (*dtrace_fasttrap_fork_ptr)(proc_t *, proc_t *);
3170Sstevel@tonic-gate void (*dtrace_fasttrap_exec_ptr)(proc_t *);
3180Sstevel@tonic-gate void (*dtrace_fasttrap_exit_ptr)(proc_t *);
3190Sstevel@tonic-gate
3200Sstevel@tonic-gate /*
3210Sstevel@tonic-gate * This function is called by cfork() in the event that it appears that
3220Sstevel@tonic-gate * there may be dtrace tracepoints active in the parent process's address
3230Sstevel@tonic-gate * space. This first confirms the existence of dtrace tracepoints in the
3240Sstevel@tonic-gate * parent process and calls into the fasttrap module to remove the
3250Sstevel@tonic-gate * corresponding tracepoints from the child. By knowing that there are
3260Sstevel@tonic-gate * existing tracepoints, and ensuring they can't be removed, we can rely
3270Sstevel@tonic-gate * on the fasttrap module remaining loaded.
3280Sstevel@tonic-gate */
3290Sstevel@tonic-gate void
dtrace_fasttrap_fork(proc_t * p,proc_t * cp)3300Sstevel@tonic-gate dtrace_fasttrap_fork(proc_t *p, proc_t *cp)
3310Sstevel@tonic-gate {
3320Sstevel@tonic-gate ASSERT(p->p_proc_flag & P_PR_LOCK);
3331880Sahl ASSERT(p->p_dtrace_count > 0);
3341880Sahl ASSERT(dtrace_fasttrap_fork_ptr != NULL);
3350Sstevel@tonic-gate
3361880Sahl dtrace_fasttrap_fork_ptr(p, cp);
3370Sstevel@tonic-gate }
338