1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #include <sys/types.h>
28 #include <sys/param.h>
29 #include <sys/systm.h>
30 #include <sys/disp.h>
31 #include <sys/var.h>
32 #include <sys/cmn_err.h>
33 #include <sys/debug.h>
34 #include <sys/x86_archext.h>
35 #include <sys/archsystm.h>
36 #include <sys/cpuvar.h>
37 #include <sys/psm_defs.h>
38 #include <sys/clock.h>
39 #include <sys/atomic.h>
40 #include <sys/lockstat.h>
41 #include <sys/smp_impldefs.h>
42 #include <sys/dtrace.h>
43 #include <sys/time.h>
44 #include <sys/panic.h>
45 #include <sys/cpu.h>
46
47 /*
48 * Using the Pentium's TSC register for gethrtime()
49 * ------------------------------------------------
50 *
51 * The Pentium family, like many chip architectures, has a high-resolution
52 * timestamp counter ("TSC") which increments once per CPU cycle. The contents
53 * of the timestamp counter are read with the RDTSC instruction.
54 *
55 * As with its UltraSPARC equivalent (the %tick register), TSC's cycle count
56 * must be translated into nanoseconds in order to implement gethrtime().
57 * We avoid inducing floating point operations in this conversion by
58 * implementing the same nsec_scale algorithm as that found in the sun4u
59 * platform code. The sun4u NATIVE_TIME_TO_NSEC_SCALE block comment contains
60 * a detailed description of the algorithm; the comment is not reproduced
61 * here. This implementation differs only in its value for NSEC_SHIFT:
62 * we implement an NSEC_SHIFT of 5 (instead of sun4u's 4) to allow for
63 * 60 MHz Pentiums.
64 *
65 * While TSC and %tick are both cycle counting registers, TSC's functionality
66 * falls short in several critical ways:
67 *
68 * (a) TSCs on different CPUs are not guaranteed to be in sync. While in
69 * practice they often _are_ in sync, this isn't guaranteed by the
70 * architecture.
71 *
72 * (b) The TSC cannot be reliably set to an arbitrary value. The architecture
73 * only supports writing the low 32-bits of TSC, making it impractical
74 * to rewrite.
75 *
76 * (c) The architecture doesn't have the capacity to interrupt based on
77 * arbitrary values of TSC; there is no TICK_CMPR equivalent.
78 *
79 * Together, (a) and (b) imply that software must track the skew between
80 * TSCs and account for it (it is assumed that while there may exist skew,
81 * there does not exist drift). To determine the skew between CPUs, we
82 * have newly onlined CPUs call tsc_sync_slave(), while the CPU performing
83 * the online operation calls tsc_sync_master().
84 *
85 * In the absence of time-of-day clock adjustments, gethrtime() must stay in
86 * sync with gettimeofday(). This is problematic; given (c), the software
87 * cannot drive its time-of-day source from TSC, and yet they must somehow be
88 * kept in sync. We implement this by having a routine, tsc_tick(), which
89 * is called once per second from the interrupt which drives time-of-day.
90 *
91 * Note that the hrtime base for gethrtime, tsc_hrtime_base, is modified
92 * atomically with nsec_scale under CLOCK_LOCK. This assures that time
93 * monotonically increases.
94 */
95
96 #define NSEC_SHIFT 5
97
98 static uint_t nsec_scale;
99 static uint_t nsec_unscale;
100
101 /*
102 * These two variables used to be grouped together inside of a structure that
103 * lived on a single cache line. A regression (bug ID 4623398) caused the
104 * compiler to emit code that "optimized" away the while-loops below. The
105 * result was that no synchronization between the onlining and onlined CPUs
106 * took place.
107 */
108 static volatile int tsc_ready;
109 static volatile int tsc_sync_go;
110
111 /*
112 * Used as indices into the tsc_sync_snaps[] array.
113 */
114 #define TSC_MASTER 0
115 #define TSC_SLAVE 1
116
117 /*
118 * Used in the tsc_master_sync()/tsc_slave_sync() rendezvous.
119 */
120 #define TSC_SYNC_STOP 1
121 #define TSC_SYNC_GO 2
122 #define TSC_SYNC_DONE 3
123 #define SYNC_ITERATIONS 10
124
125 #define TSC_CONVERT_AND_ADD(tsc, hrt, scale) { \
126 unsigned int *_l = (unsigned int *)&(tsc); \
127 (hrt) += mul32(_l[1], scale) << NSEC_SHIFT; \
128 (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \
129 }
130
131 #define TSC_CONVERT(tsc, hrt, scale) { \
132 unsigned int *_l = (unsigned int *)&(tsc); \
133 (hrt) = mul32(_l[1], scale) << NSEC_SHIFT; \
134 (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \
135 }
136
137 int tsc_master_slave_sync_needed = 1;
138
139 static int tsc_max_delta;
140 static hrtime_t tsc_sync_tick_delta[NCPU];
141 typedef struct tsc_sync {
142 volatile hrtime_t master_tsc, slave_tsc;
143 } tsc_sync_t;
144 static tsc_sync_t *tscp;
145 static hrtime_t largest_tsc_delta = 0;
146 static ulong_t shortest_write_time = ~0UL;
147
148 static hrtime_t tsc_last = 0;
149 static hrtime_t tsc_last_jumped = 0;
150 static hrtime_t tsc_hrtime_base = 0;
151 static int tsc_jumped = 0;
152
153 static hrtime_t shadow_tsc_hrtime_base;
154 static hrtime_t shadow_tsc_last;
155 static uint_t shadow_nsec_scale;
156 static uint32_t shadow_hres_lock;
157 int get_tsc_ready();
158
159 hrtime_t
tsc_gethrtime(void)160 tsc_gethrtime(void)
161 {
162 uint32_t old_hres_lock;
163 hrtime_t tsc, hrt;
164
165 do {
166 old_hres_lock = hres_lock;
167
168 if ((tsc = tsc_read()) >= tsc_last) {
169 /*
170 * It would seem to be obvious that this is true
171 * (that is, the past is less than the present),
172 * but it isn't true in the presence of suspend/resume
173 * cycles. If we manage to call gethrtime()
174 * after a resume, but before the first call to
175 * tsc_tick(), we will see the jump. In this case,
176 * we will simply use the value in TSC as the delta.
177 */
178 tsc -= tsc_last;
179 } else if (tsc >= tsc_last - 2*tsc_max_delta) {
180 /*
181 * There is a chance that tsc_tick() has just run on
182 * another CPU, and we have drifted just enough so that
183 * we appear behind tsc_last. In this case, force the
184 * delta to be zero.
185 */
186 tsc = 0;
187 }
188
189 hrt = tsc_hrtime_base;
190
191 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
192 } while ((old_hres_lock & ~1) != hres_lock);
193
194 return (hrt);
195 }
196
197 hrtime_t
tsc_gethrtime_delta(void)198 tsc_gethrtime_delta(void)
199 {
200 uint32_t old_hres_lock;
201 hrtime_t tsc, hrt;
202 ulong_t flags;
203
204 do {
205 old_hres_lock = hres_lock;
206
207 /*
208 * We need to disable interrupts here to assure that we
209 * don't migrate between the call to tsc_read() and
210 * adding the CPU's TSC tick delta. Note that disabling
211 * and reenabling preemption is forbidden here because
212 * we may be in the middle of a fast trap. In the amd64
213 * kernel we cannot tolerate preemption during a fast
214 * trap. See _update_sregs().
215 */
216
217 flags = clear_int_flag();
218 tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id];
219 restore_int_flag(flags);
220
221 /* See comments in tsc_gethrtime() above */
222
223 if (tsc >= tsc_last) {
224 tsc -= tsc_last;
225 } else if (tsc >= tsc_last - 2 * tsc_max_delta) {
226 tsc = 0;
227 }
228
229 hrt = tsc_hrtime_base;
230
231 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
232 } while ((old_hres_lock & ~1) != hres_lock);
233
234 return (hrt);
235 }
236
237 /*
238 * This is similar to the above, but it cannot actually spin on hres_lock.
239 * As a result, it caches all of the variables it needs; if the variables
240 * don't change, it's done.
241 */
242 hrtime_t
dtrace_gethrtime(void)243 dtrace_gethrtime(void)
244 {
245 uint32_t old_hres_lock;
246 hrtime_t tsc, hrt;
247 ulong_t flags;
248
249 do {
250 old_hres_lock = hres_lock;
251
252 /*
253 * Interrupts are disabled to ensure that the thread isn't
254 * migrated between the tsc_read() and adding the CPU's
255 * TSC tick delta.
256 */
257 flags = clear_int_flag();
258
259 tsc = tsc_read();
260
261 if (gethrtimef == tsc_gethrtime_delta)
262 tsc += tsc_sync_tick_delta[CPU->cpu_id];
263
264 restore_int_flag(flags);
265
266 /*
267 * See the comments in tsc_gethrtime(), above.
268 */
269 if (tsc >= tsc_last)
270 tsc -= tsc_last;
271 else if (tsc >= tsc_last - 2*tsc_max_delta)
272 tsc = 0;
273
274 hrt = tsc_hrtime_base;
275
276 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
277
278 if ((old_hres_lock & ~1) == hres_lock)
279 break;
280
281 /*
282 * If we're here, the clock lock is locked -- or it has been
283 * unlocked and locked since we looked. This may be due to
284 * tsc_tick() running on another CPU -- or it may be because
285 * some code path has ended up in dtrace_probe() with
286 * CLOCK_LOCK held. We'll try to determine that we're in
287 * the former case by taking another lap if the lock has
288 * changed since when we first looked at it.
289 */
290 if (old_hres_lock != hres_lock)
291 continue;
292
293 /*
294 * So the lock was and is locked. We'll use the old data
295 * instead.
296 */
297 old_hres_lock = shadow_hres_lock;
298
299 /*
300 * Again, disable interrupts to ensure that the thread
301 * isn't migrated between the tsc_read() and adding
302 * the CPU's TSC tick delta.
303 */
304 flags = clear_int_flag();
305
306 tsc = tsc_read();
307
308 if (gethrtimef == tsc_gethrtime_delta)
309 tsc += tsc_sync_tick_delta[CPU->cpu_id];
310
311 restore_int_flag(flags);
312
313 /*
314 * See the comments in tsc_gethrtime(), above.
315 */
316 if (tsc >= shadow_tsc_last)
317 tsc -= shadow_tsc_last;
318 else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta)
319 tsc = 0;
320
321 hrt = shadow_tsc_hrtime_base;
322
323 TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale);
324 } while ((old_hres_lock & ~1) != shadow_hres_lock);
325
326 return (hrt);
327 }
328
329 hrtime_t
tsc_gethrtimeunscaled(void)330 tsc_gethrtimeunscaled(void)
331 {
332 uint32_t old_hres_lock;
333 hrtime_t tsc;
334
335 do {
336 old_hres_lock = hres_lock;
337
338 /* See tsc_tick(). */
339 tsc = tsc_read() + tsc_last_jumped;
340 } while ((old_hres_lock & ~1) != hres_lock);
341
342 return (tsc);
343 }
344
345 /*
346 * Convert a nanosecond based timestamp to tsc
347 */
348 uint64_t
tsc_unscalehrtime(hrtime_t nsec)349 tsc_unscalehrtime(hrtime_t nsec)
350 {
351 hrtime_t tsc;
352
353 if (tsc_gethrtime_enable) {
354 TSC_CONVERT(nsec, tsc, nsec_unscale);
355 return (tsc);
356 }
357 return ((uint64_t)nsec);
358 }
359
360 /* Convert a tsc timestamp to nanoseconds */
361 void
tsc_scalehrtime(hrtime_t * tsc)362 tsc_scalehrtime(hrtime_t *tsc)
363 {
364 hrtime_t hrt;
365 hrtime_t mytsc;
366
367 if (tsc == NULL)
368 return;
369 mytsc = *tsc;
370
371 TSC_CONVERT(mytsc, hrt, nsec_scale);
372 *tsc = hrt;
373 }
374
375 hrtime_t
tsc_gethrtimeunscaled_delta(void)376 tsc_gethrtimeunscaled_delta(void)
377 {
378 hrtime_t hrt;
379 ulong_t flags;
380
381 /*
382 * Similarly to tsc_gethrtime_delta, we need to disable preemption
383 * to prevent migration between the call to tsc_gethrtimeunscaled
384 * and adding the CPU's hrtime delta. Note that disabling and
385 * reenabling preemption is forbidden here because we may be in the
386 * middle of a fast trap. In the amd64 kernel we cannot tolerate
387 * preemption during a fast trap. See _update_sregs().
388 */
389
390 flags = clear_int_flag();
391 hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id];
392 restore_int_flag(flags);
393
394 return (hrt);
395 }
396
397 /*
398 * Called by the master in the TSC sync operation (usually the boot CPU).
399 * If the slave is discovered to have a skew, gethrtimef will be changed to
400 * point to tsc_gethrtime_delta(). Calculating skews is precise only when
401 * the master and slave TSCs are read simultaneously; however, there is no
402 * algorithm that can read both CPUs in perfect simultaneity. The proposed
403 * algorithm is an approximate method based on the behaviour of cache
404 * management. The slave CPU continuously reads TSC and then reads a global
405 * variable which the master CPU updates. The moment the master's update reaches
406 * the slave's visibility (being forced by an mfence operation) we use the TSC
407 * reading taken on the slave. A corresponding TSC read will be taken on the
408 * master as soon as possible after finishing the mfence operation. But the
409 * delay between causing the slave to notice the invalid cache line and the
410 * competion of mfence is not repeatable. This error is heuristically assumed
411 * to be 1/4th of the total write time as being measured by the two TSC reads
412 * on the master sandwiching the mfence. Furthermore, due to the nature of
413 * bus arbitration, contention on memory bus, etc., the time taken for the write
414 * to reflect globally can vary a lot. So instead of taking a single reading,
415 * a set of readings are taken and the one with least write time is chosen
416 * to calculate the final skew.
417 *
418 * TSC sync is disabled in the context of virtualization because the CPUs
419 * assigned to the guest are virtual CPUs which means the real CPUs on which
420 * guest runs keep changing during life time of guest OS. So we would end up
421 * calculating TSC skews for a set of CPUs during boot whereas the guest
422 * might migrate to a different set of physical CPUs at a later point of
423 * time.
424 */
425 void
tsc_sync_master(processorid_t slave)426 tsc_sync_master(processorid_t slave)
427 {
428 ulong_t flags, source, min_write_time = ~0UL;
429 hrtime_t write_time, x, mtsc_after, tdelta;
430 tsc_sync_t *tsc = tscp;
431 int cnt;
432 int hwtype;
433
434 hwtype = get_hwenv();
435 if (!tsc_master_slave_sync_needed || hwtype == HW_XEN_HVM ||
436 hwtype == HW_VMWARE)
437 return;
438
439 flags = clear_int_flag();
440 source = CPU->cpu_id;
441
442 for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) {
443 while (tsc_sync_go != TSC_SYNC_GO)
444 SMT_PAUSE();
445
446 tsc->master_tsc = tsc_read();
447 membar_enter();
448 mtsc_after = tsc_read();
449 while (tsc_sync_go != TSC_SYNC_DONE)
450 SMT_PAUSE();
451 write_time = mtsc_after - tsc->master_tsc;
452 if (write_time <= min_write_time) {
453 min_write_time = write_time;
454 /*
455 * Apply heuristic adjustment only if the calculated
456 * delta is > 1/4th of the write time.
457 */
458 x = tsc->slave_tsc - mtsc_after;
459 if (x < 0)
460 x = -x;
461 if (x > (min_write_time/4))
462 /*
463 * Subtract 1/4th of the measured write time
464 * from the master's TSC value, as an estimate
465 * of how late the mfence completion came
466 * after the slave noticed the cache line
467 * change.
468 */
469 tdelta = tsc->slave_tsc -
470 (mtsc_after - (min_write_time/4));
471 else
472 tdelta = tsc->slave_tsc - mtsc_after;
473 tsc_sync_tick_delta[slave] =
474 tsc_sync_tick_delta[source] - tdelta;
475 }
476
477 tsc->master_tsc = tsc->slave_tsc = write_time = 0;
478 membar_enter();
479 tsc_sync_go = TSC_SYNC_STOP;
480 }
481 if (tdelta < 0)
482 tdelta = -tdelta;
483 if (tdelta > largest_tsc_delta)
484 largest_tsc_delta = tdelta;
485 if (min_write_time < shortest_write_time)
486 shortest_write_time = min_write_time;
487 /*
488 * Enable delta variants of tsc functions if the largest of all chosen
489 * deltas is > smallest of the write time.
490 */
491 if (largest_tsc_delta > shortest_write_time) {
492 gethrtimef = tsc_gethrtime_delta;
493 gethrtimeunscaledf = tsc_gethrtimeunscaled_delta;
494 }
495 restore_int_flag(flags);
496 }
497
498 /*
499 * Called by a CPU which has just been onlined. It is expected that the CPU
500 * performing the online operation will call tsc_sync_master().
501 *
502 * TSC sync is disabled in the context of virtualization. See comments
503 * above tsc_sync_master.
504 */
505 void
tsc_sync_slave(void)506 tsc_sync_slave(void)
507 {
508 ulong_t flags;
509 hrtime_t s1;
510 tsc_sync_t *tsc = tscp;
511 int cnt;
512 int hwtype;
513
514 hwtype = get_hwenv();
515 if (!tsc_master_slave_sync_needed || hwtype == HW_XEN_HVM ||
516 hwtype == HW_VMWARE)
517 return;
518
519 flags = clear_int_flag();
520
521 for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) {
522 /* Re-fill the cache line */
523 s1 = tsc->master_tsc;
524 membar_enter();
525 tsc_sync_go = TSC_SYNC_GO;
526 do {
527 /*
528 * Do not put an SMT_PAUSE here. For instance,
529 * if the master and slave are really the same
530 * hyper-threaded CPU, then you want the master
531 * to yield to the slave as quickly as possible here,
532 * but not the other way.
533 */
534 s1 = tsc_read();
535 } while (tsc->master_tsc == 0);
536 tsc->slave_tsc = s1;
537 membar_enter();
538 tsc_sync_go = TSC_SYNC_DONE;
539
540 while (tsc_sync_go != TSC_SYNC_STOP)
541 SMT_PAUSE();
542 }
543
544 restore_int_flag(flags);
545 }
546
547 /*
548 * Called once per second on a CPU from the cyclic subsystem's
549 * CY_HIGH_LEVEL interrupt. (No longer just cpu0-only)
550 */
551 void
tsc_tick(void)552 tsc_tick(void)
553 {
554 hrtime_t now, delta;
555 ushort_t spl;
556
557 /*
558 * Before we set the new variables, we set the shadow values. This
559 * allows for lock free operation in dtrace_gethrtime().
560 */
561 lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET,
562 ipltospl(CBE_HIGH_PIL), &spl);
563
564 shadow_tsc_hrtime_base = tsc_hrtime_base;
565 shadow_tsc_last = tsc_last;
566 shadow_nsec_scale = nsec_scale;
567
568 shadow_hres_lock++;
569 splx(spl);
570
571 CLOCK_LOCK(&spl);
572
573 now = tsc_read();
574
575 if (gethrtimef == tsc_gethrtime_delta)
576 now += tsc_sync_tick_delta[CPU->cpu_id];
577
578 if (now < tsc_last) {
579 /*
580 * The TSC has just jumped into the past. We assume that
581 * this is due to a suspend/resume cycle, and we're going
582 * to use the _current_ value of TSC as the delta. This
583 * will keep tsc_hrtime_base correct. We're also going to
584 * assume that rate of tsc does not change after a suspend
585 * resume (i.e nsec_scale remains the same).
586 */
587 delta = now;
588 tsc_last_jumped += tsc_last;
589 tsc_jumped = 1;
590 } else {
591 /*
592 * Determine the number of TSC ticks since the last clock
593 * tick, and add that to the hrtime base.
594 */
595 delta = now - tsc_last;
596 }
597
598 TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale);
599 tsc_last = now;
600
601 CLOCK_UNLOCK(spl);
602 }
603
604 void
tsc_hrtimeinit(uint64_t cpu_freq_hz)605 tsc_hrtimeinit(uint64_t cpu_freq_hz)
606 {
607 extern int gethrtime_hires;
608 longlong_t tsc;
609 ulong_t flags;
610
611 /*
612 * cpu_freq_hz is the measured cpu frequency in hertz
613 */
614
615 /*
616 * We can't accommodate CPUs slower than 31.25 MHz.
617 */
618 ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT));
619 nsec_scale =
620 (uint_t)(((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz);
621 nsec_unscale =
622 (uint_t)(((uint64_t)cpu_freq_hz << (32 - NSEC_SHIFT)) / NANOSEC);
623
624 flags = clear_int_flag();
625 tsc = tsc_read();
626 (void) tsc_gethrtime();
627 tsc_max_delta = tsc_read() - tsc;
628 restore_int_flag(flags);
629 gethrtimef = tsc_gethrtime;
630 gethrtimeunscaledf = tsc_gethrtimeunscaled;
631 scalehrtimef = tsc_scalehrtime;
632 unscalehrtimef = tsc_unscalehrtime;
633 hrtime_tick = tsc_tick;
634 gethrtime_hires = 1;
635 /*
636 * Allocate memory for the structure used in the tsc sync logic.
637 * This structure should be aligned on a multiple of cache line size.
638 */
639 tscp = kmem_zalloc(PAGESIZE, KM_SLEEP);
640 }
641
642 int
get_tsc_ready()643 get_tsc_ready()
644 {
645 return (tsc_ready);
646 }
647
648 /*
649 * Adjust all the deltas by adding the passed value to the array.
650 * Then use the "delt" versions of the the gethrtime functions.
651 * Note that 'tdelta' _could_ be a negative number, which should
652 * reduce the values in the array (used, for example, if the Solaris
653 * instance was moved by a virtual manager to a machine with a higher
654 * value of tsc).
655 */
656 void
tsc_adjust_delta(hrtime_t tdelta)657 tsc_adjust_delta(hrtime_t tdelta)
658 {
659 int i;
660
661 for (i = 0; i < NCPU; i++) {
662 tsc_sync_tick_delta[i] += tdelta;
663 }
664
665 gethrtimef = tsc_gethrtime_delta;
666 gethrtimeunscaledf = tsc_gethrtimeunscaled_delta;
667 }
668
669 /*
670 * Functions to manage TSC and high-res time on suspend and resume.
671 */
672
673 /*
674 * declarations needed for time adjustment
675 */
676 extern void rtcsync(void);
677 extern tod_ops_t *tod_ops;
678 /* There must be a better way than exposing nsec_scale! */
679 extern uint_t nsec_scale;
680 static uint64_t tsc_saved_tsc = 0; /* 1 in 2^64 chance this'll screw up! */
681 static timestruc_t tsc_saved_ts;
682 static int tsc_needs_resume = 0; /* We only want to do this once. */
683 int tsc_delta_onsuspend = 0;
684 int tsc_adjust_seconds = 1;
685 int tsc_suspend_count = 0;
686 int tsc_resume_in_cyclic = 0;
687
688 /*
689 * Let timestamp.c know that we are suspending. It needs to take
690 * snapshots of the current time, and do any pre-suspend work.
691 */
692 void
tsc_suspend(void)693 tsc_suspend(void)
694 {
695 /*
696 * What we need to do here, is to get the time we suspended, so that we
697 * know how much we should add to the resume.
698 * This routine is called by each CPU, so we need to handle reentry.
699 */
700 if (tsc_gethrtime_enable) {
701 /*
702 * We put the tsc_read() inside the lock as it
703 * as no locking constraints, and it puts the
704 * aquired value closer to the time stamp (in
705 * case we delay getting the lock).
706 */
707 mutex_enter(&tod_lock);
708 tsc_saved_tsc = tsc_read();
709 tsc_saved_ts = TODOP_GET(tod_ops);
710 mutex_exit(&tod_lock);
711 /* We only want to do this once. */
712 if (tsc_needs_resume == 0) {
713 if (tsc_delta_onsuspend) {
714 tsc_adjust_delta(tsc_saved_tsc);
715 } else {
716 tsc_adjust_delta(nsec_scale);
717 }
718 tsc_suspend_count++;
719 }
720 }
721
722 invalidate_cache();
723 tsc_needs_resume = 1;
724 }
725
726 /*
727 * Restore all timestamp state based on the snapshots taken at
728 * suspend time.
729 */
730 void
tsc_resume(void)731 tsc_resume(void)
732 {
733 /*
734 * We only need to (and want to) do this once. So let the first
735 * caller handle this (we are locked by the cpu lock), as it
736 * is preferential that we get the earliest sync.
737 */
738 if (tsc_needs_resume) {
739 /*
740 * If using the TSC, adjust the delta based on how long
741 * we were sleeping (or away). We also adjust for
742 * migration and a grown TSC.
743 */
744 if (tsc_saved_tsc != 0) {
745 timestruc_t ts;
746 hrtime_t now, sleep_tsc = 0;
747 int sleep_sec;
748 extern void tsc_tick(void);
749 extern uint64_t cpu_freq_hz;
750
751 /* tsc_read() MUST be before TODOP_GET() */
752 mutex_enter(&tod_lock);
753 now = tsc_read();
754 ts = TODOP_GET(tod_ops);
755 mutex_exit(&tod_lock);
756
757 /* Compute seconds of sleep time */
758 sleep_sec = ts.tv_sec - tsc_saved_ts.tv_sec;
759
760 /*
761 * If the saved sec is less that or equal to
762 * the current ts, then there is likely a
763 * problem with the clock. Assume at least
764 * one second has passed, so that time goes forward.
765 */
766 if (sleep_sec <= 0) {
767 sleep_sec = 1;
768 }
769
770 /* How many TSC's should have occured while sleeping */
771 if (tsc_adjust_seconds)
772 sleep_tsc = sleep_sec * cpu_freq_hz;
773
774 /*
775 * We also want to subtract from the "sleep_tsc"
776 * the current value of tsc_read(), so that our
777 * adjustment accounts for the amount of time we
778 * have been resumed _or_ an adjustment based on
779 * the fact that we didn't actually power off the
780 * CPU (migration is another issue, but _should_
781 * also comply with this calculation). If the CPU
782 * never powered off, then:
783 * 'now == sleep_tsc + saved_tsc'
784 * and the delta will effectively be "0".
785 */
786 sleep_tsc -= now;
787 if (tsc_delta_onsuspend) {
788 tsc_adjust_delta(sleep_tsc);
789 } else {
790 tsc_adjust_delta(tsc_saved_tsc + sleep_tsc);
791 }
792 tsc_saved_tsc = 0;
793
794 tsc_tick();
795 }
796 tsc_needs_resume = 0;
797 }
798
799 }
800