1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24 /*
25 * Copyright (c) 2009, Intel Corporation.
26 * All Rights Reserved.
27 */
28
29 #include <sys/x86_archext.h>
30 #include <sys/machsystm.h>
31 #include <sys/archsystm.h>
32 #include <sys/x_call.h>
33 #include <sys/acpi/acpi.h>
34 #include <sys/acpica.h>
35 #include <sys/speedstep.h>
36 #include <sys/cpu_acpi.h>
37 #include <sys/cpupm.h>
38 #include <sys/dtrace.h>
39 #include <sys/sdt.h>
40
41 /*
42 * turbo related structure definitions
43 */
44 typedef struct cpupm_turbo_info {
45 kstat_t *turbo_ksp; /* turbo kstat */
46 int in_turbo; /* in turbo? */
47 int turbo_supported; /* turbo flag */
48 uint64_t t_mcnt; /* turbo mcnt */
49 uint64_t t_acnt; /* turbo acnt */
50 } cpupm_turbo_info_t;
51
52 typedef struct turbo_kstat_s {
53 struct kstat_named turbo_supported; /* turbo flag */
54 struct kstat_named t_mcnt; /* IA32_MPERF_MSR */
55 struct kstat_named t_acnt; /* IA32_APERF_MSR */
56 } turbo_kstat_t;
57
58 static int speedstep_init(cpu_t *);
59 static void speedstep_fini(cpu_t *);
60 static void speedstep_power(cpuset_t, uint32_t);
61 static void speedstep_stop(cpu_t *);
62 static boolean_t turbo_supported(void);
63 static int turbo_kstat_update(kstat_t *, int);
64 static void get_turbo_info(cpupm_turbo_info_t *);
65 static void reset_turbo_info(void);
66 static void record_turbo_info(cpupm_turbo_info_t *, uint32_t, uint32_t);
67 static void update_turbo_info(cpupm_turbo_info_t *);
68
69 /*
70 * Interfaces for modules implementing Intel's Enhanced SpeedStep.
71 */
72 cpupm_state_ops_t speedstep_ops = {
73 "Enhanced SpeedStep Technology",
74 speedstep_init,
75 speedstep_fini,
76 speedstep_power,
77 speedstep_stop
78 };
79
80 /*
81 * Error returns
82 */
83 #define ESS_RET_SUCCESS 0x00
84 #define ESS_RET_NO_PM 0x01
85 #define ESS_RET_UNSUP_STATE 0x02
86
87 /*
88 * MSR registers for changing and reading processor power state.
89 */
90 #define IA32_PERF_STAT_MSR 0x198
91 #define IA32_PERF_CTL_MSR 0x199
92
93 #define IA32_CPUID_TSC_CONSTANT 0xF30
94 #define IA32_MISC_ENABLE_MSR 0x1A0
95 #define IA32_MISC_ENABLE_EST (1<<16)
96 #define IA32_MISC_ENABLE_CXE (1<<25)
97
98 #define CPUID_TURBO_SUPPORT (1 << 1)
99 #define CPU_ACPI_P0 0
100 #define CPU_IN_TURBO 1
101
102 /*
103 * MSR for hardware coordination feedback mechanism
104 * - IA32_MPERF: increments in proportion to a fixed frequency
105 * - IA32_APERF: increments in proportion to actual performance
106 */
107 #define IA32_MPERF_MSR 0xE7
108 #define IA32_APERF_MSR 0xE8
109
110 /*
111 * Debugging support
112 */
113 #ifdef DEBUG
114 volatile int ess_debug = 0;
115 #define ESSDEBUG(arglist) if (ess_debug) printf arglist;
116 #else
117 #define ESSDEBUG(arglist)
118 #endif
119
120 static kmutex_t turbo_mutex;
121
122 turbo_kstat_t turbo_kstat = {
123 { "turbo_supported", KSTAT_DATA_UINT32 },
124 { "turbo_mcnt", KSTAT_DATA_UINT64 },
125 { "turbo_acnt", KSTAT_DATA_UINT64 },
126 };
127
128 /*
129 * kstat update function of the turbo mode info
130 */
131 static int
turbo_kstat_update(kstat_t * ksp,int flag)132 turbo_kstat_update(kstat_t *ksp, int flag)
133 {
134 cpupm_turbo_info_t *turbo_info = ksp->ks_private;
135
136 if (flag == KSTAT_WRITE) {
137 return (EACCES);
138 }
139
140 /*
141 * update the count in case CPU is in the turbo
142 * mode for a long time
143 */
144 if (turbo_info->in_turbo == CPU_IN_TURBO)
145 update_turbo_info(turbo_info);
146
147 turbo_kstat.turbo_supported.value.ui32 =
148 turbo_info->turbo_supported;
149 turbo_kstat.t_mcnt.value.ui64 = turbo_info->t_mcnt;
150 turbo_kstat.t_acnt.value.ui64 = turbo_info->t_acnt;
151
152 return (0);
153 }
154
155 /*
156 * Get count of MPERF/APERF MSR
157 */
158 static void
get_turbo_info(cpupm_turbo_info_t * turbo_info)159 get_turbo_info(cpupm_turbo_info_t *turbo_info)
160 {
161 ulong_t iflag;
162 uint64_t mcnt, acnt;
163
164 iflag = intr_clear();
165 mcnt = rdmsr(IA32_MPERF_MSR);
166 acnt = rdmsr(IA32_APERF_MSR);
167 turbo_info->t_mcnt += mcnt;
168 turbo_info->t_acnt += acnt;
169 intr_restore(iflag);
170 }
171
172 /*
173 * Clear MPERF/APERF MSR
174 */
175 static void
reset_turbo_info(void)176 reset_turbo_info(void)
177 {
178 ulong_t iflag;
179
180 iflag = intr_clear();
181 wrmsr(IA32_MPERF_MSR, 0);
182 wrmsr(IA32_APERF_MSR, 0);
183 intr_restore(iflag);
184 }
185
186 /*
187 * sum up the count of one CPU_ACPI_P0 transition
188 */
189 static void
record_turbo_info(cpupm_turbo_info_t * turbo_info,uint32_t cur_state,uint32_t req_state)190 record_turbo_info(cpupm_turbo_info_t *turbo_info,
191 uint32_t cur_state, uint32_t req_state)
192 {
193 if (!turbo_info->turbo_supported)
194 return;
195 /*
196 * enter P0 state
197 */
198 if (req_state == CPU_ACPI_P0) {
199 reset_turbo_info();
200 turbo_info->in_turbo = CPU_IN_TURBO;
201 }
202 /*
203 * Leave P0 state
204 */
205 else if (cur_state == CPU_ACPI_P0) {
206 turbo_info->in_turbo = 0;
207 get_turbo_info(turbo_info);
208 }
209 }
210
211 /*
212 * update the sum of counts and clear MSRs
213 */
214 static void
update_turbo_info(cpupm_turbo_info_t * turbo_info)215 update_turbo_info(cpupm_turbo_info_t *turbo_info)
216 {
217 ulong_t iflag;
218 uint64_t mcnt, acnt;
219
220 iflag = intr_clear();
221 mcnt = rdmsr(IA32_MPERF_MSR);
222 acnt = rdmsr(IA32_APERF_MSR);
223 wrmsr(IA32_MPERF_MSR, 0);
224 wrmsr(IA32_APERF_MSR, 0);
225 turbo_info->t_mcnt += mcnt;
226 turbo_info->t_acnt += acnt;
227 intr_restore(iflag);
228 }
229
230 /*
231 * Write the ctrl register. How it is written, depends upon the _PCT
232 * APCI object value.
233 */
234 static void
write_ctrl(cpu_acpi_handle_t handle,uint32_t ctrl)235 write_ctrl(cpu_acpi_handle_t handle, uint32_t ctrl)
236 {
237 cpu_acpi_pct_t *pct_ctrl;
238 uint64_t reg;
239
240 pct_ctrl = CPU_ACPI_PCT_CTRL(handle);
241
242 switch (pct_ctrl->cr_addrspace_id) {
243 case ACPI_ADR_SPACE_FIXED_HARDWARE:
244 /*
245 * Read current power state because reserved bits must be
246 * preserved, compose new value, and write it.
247 */
248 reg = rdmsr(IA32_PERF_CTL_MSR);
249 reg &= ~((uint64_t)0xFFFF);
250 reg |= ctrl;
251 wrmsr(IA32_PERF_CTL_MSR, reg);
252 break;
253
254 case ACPI_ADR_SPACE_SYSTEM_IO:
255 (void) cpu_acpi_write_port(pct_ctrl->cr_address, ctrl,
256 pct_ctrl->cr_width);
257 break;
258
259 default:
260 DTRACE_PROBE1(ess_ctrl_unsupported_type, uint8_t,
261 pct_ctrl->cr_addrspace_id);
262 return;
263 }
264
265 DTRACE_PROBE1(ess_ctrl_write, uint32_t, ctrl);
266 }
267
268 /*
269 * Transition the current processor to the requested state.
270 */
271 void
speedstep_pstate_transition(uint32_t req_state)272 speedstep_pstate_transition(uint32_t req_state)
273 {
274 cpupm_mach_state_t *mach_state =
275 (cpupm_mach_state_t *)CPU->cpu_m.mcpu_pm_mach_state;
276 cpu_acpi_handle_t handle = mach_state->ms_acpi_handle;
277 cpu_acpi_pstate_t *req_pstate;
278 uint32_t ctrl;
279 cpupm_turbo_info_t *turbo_info =
280 (cpupm_turbo_info_t *)(mach_state->ms_vendor);
281
282 req_pstate = (cpu_acpi_pstate_t *)CPU_ACPI_PSTATES(handle);
283 req_pstate += req_state;
284
285 DTRACE_PROBE1(ess_transition, uint32_t, CPU_ACPI_FREQ(req_pstate));
286
287 /*
288 * Initiate the processor p-state change.
289 */
290 ctrl = CPU_ACPI_PSTATE_CTRL(req_pstate);
291 write_ctrl(handle, ctrl);
292
293 if (turbo_info)
294 record_turbo_info(turbo_info,
295 mach_state->ms_pstate.cma_state.pstate, req_state);
296
297
298 mach_state->ms_pstate.cma_state.pstate = req_state;
299 cpu_set_curr_clock(((uint64_t)CPU_ACPI_FREQ(req_pstate) * 1000000));
300 }
301
302 static void
speedstep_power(cpuset_t set,uint32_t req_state)303 speedstep_power(cpuset_t set, uint32_t req_state)
304 {
305 /*
306 * If thread is already running on target CPU then just
307 * make the transition request. Otherwise, we'll need to
308 * make a cross-call.
309 */
310 kpreempt_disable();
311 if (CPU_IN_SET(set, CPU->cpu_id)) {
312 speedstep_pstate_transition(req_state);
313 CPUSET_DEL(set, CPU->cpu_id);
314 }
315 if (!CPUSET_ISNULL(set)) {
316 xc_call((xc_arg_t)req_state, NULL, NULL, CPUSET2BV(set),
317 (xc_func_t)speedstep_pstate_transition);
318 }
319 kpreempt_enable();
320 }
321
322 /*
323 * Validate that this processor supports Speedstep and if so,
324 * get the P-state data from ACPI and cache it.
325 */
326 static int
speedstep_init(cpu_t * cp)327 speedstep_init(cpu_t *cp)
328 {
329 cpupm_mach_state_t *mach_state =
330 (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state;
331 cpu_acpi_handle_t handle = mach_state->ms_acpi_handle;
332 cpu_acpi_pct_t *pct_stat;
333 cpupm_turbo_info_t *turbo_info;
334
335 ESSDEBUG(("speedstep_init: processor %d\n", cp->cpu_id));
336
337 /*
338 * Cache the P-state specific ACPI data.
339 */
340 if (cpu_acpi_cache_pstate_data(handle) != 0) {
341 cmn_err(CE_NOTE, "!SpeedStep support is being "
342 "disabled due to errors parsing ACPI P-state objects "
343 "exported by BIOS.");
344 speedstep_fini(cp);
345 return (ESS_RET_NO_PM);
346 }
347
348 pct_stat = CPU_ACPI_PCT_STATUS(handle);
349 switch (pct_stat->cr_addrspace_id) {
350 case ACPI_ADR_SPACE_FIXED_HARDWARE:
351 ESSDEBUG(("Transitions will use fixed hardware\n"));
352 break;
353 case ACPI_ADR_SPACE_SYSTEM_IO:
354 ESSDEBUG(("Transitions will use system IO\n"));
355 break;
356 default:
357 cmn_err(CE_WARN, "!_PCT conifgured for unsupported "
358 "addrspace = %d.", pct_stat->cr_addrspace_id);
359 cmn_err(CE_NOTE, "!CPU power management will not function.");
360 speedstep_fini(cp);
361 return (ESS_RET_NO_PM);
362 }
363
364 cpupm_alloc_domains(cp, CPUPM_P_STATES);
365
366 if (!turbo_supported()) {
367 mach_state->ms_vendor = NULL;
368 goto ess_ret_success;
369 }
370 /*
371 * turbo mode supported
372 */
373 turbo_info = mach_state->ms_vendor =
374 kmem_zalloc(sizeof (cpupm_turbo_info_t), KM_SLEEP);
375 turbo_info->turbo_supported = 1;
376 turbo_info->turbo_ksp = kstat_create("turbo", cp->cpu_id,
377 "turbo", "misc", KSTAT_TYPE_NAMED,
378 sizeof (turbo_kstat) / sizeof (kstat_named_t),
379 KSTAT_FLAG_VIRTUAL);
380
381 if (turbo_info->turbo_ksp == NULL) {
382 cmn_err(CE_NOTE, "kstat_create(turbo) fail");
383 } else {
384 turbo_info->turbo_ksp->ks_data = &turbo_kstat;
385 turbo_info->turbo_ksp->ks_lock = &turbo_mutex;
386 turbo_info->turbo_ksp->ks_update = turbo_kstat_update;
387 turbo_info->turbo_ksp->ks_data_size += MAXNAMELEN;
388 turbo_info->turbo_ksp->ks_private = turbo_info;
389
390 kstat_install(turbo_info->turbo_ksp);
391 }
392
393 ess_ret_success:
394
395 ESSDEBUG(("Processor %d succeeded.\n", cp->cpu_id))
396 return (ESS_RET_SUCCESS);
397 }
398
399 /*
400 * Free resources allocated by speedstep_init().
401 */
402 static void
speedstep_fini(cpu_t * cp)403 speedstep_fini(cpu_t *cp)
404 {
405 cpupm_mach_state_t *mach_state =
406 (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state);
407 cpu_acpi_handle_t handle = mach_state->ms_acpi_handle;
408 cpupm_turbo_info_t *turbo_info =
409 (cpupm_turbo_info_t *)(mach_state->ms_vendor);
410
411 cpupm_free_domains(&cpupm_pstate_domains);
412 cpu_acpi_free_pstate_data(handle);
413
414 if (turbo_info) {
415 if (turbo_info->turbo_ksp != NULL)
416 kstat_delete(turbo_info->turbo_ksp);
417 kmem_free(turbo_info, sizeof (cpupm_turbo_info_t));
418 }
419 }
420
421 static void
speedstep_stop(cpu_t * cp)422 speedstep_stop(cpu_t *cp)
423 {
424 cpupm_mach_state_t *mach_state =
425 (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state);
426 cpu_acpi_handle_t handle = mach_state->ms_acpi_handle;
427 cpupm_turbo_info_t *turbo_info =
428 (cpupm_turbo_info_t *)(mach_state->ms_vendor);
429
430 cpupm_remove_domains(cp, CPUPM_P_STATES, &cpupm_pstate_domains);
431 cpu_acpi_free_pstate_data(handle);
432
433 if (turbo_info) {
434 if (turbo_info->turbo_ksp != NULL)
435 kstat_delete(turbo_info->turbo_ksp);
436 kmem_free(turbo_info, sizeof (cpupm_turbo_info_t));
437 }
438 }
439
440 boolean_t
speedstep_supported(uint_t family,uint_t model)441 speedstep_supported(uint_t family, uint_t model)
442 {
443 struct cpuid_regs cpu_regs;
444
445 /* Required features */
446 if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
447 !is_x86_feature(x86_featureset, X86FSET_MSR)) {
448 return (B_FALSE);
449 }
450
451 /*
452 * We only support family/model combinations which
453 * are P-state TSC invariant.
454 */
455 if (!((family == 0xf && model >= 0x3) ||
456 (family == 0x6 && model >= 0xe))) {
457 return (B_FALSE);
458 }
459
460 /*
461 * Enhanced SpeedStep supported?
462 */
463 cpu_regs.cp_eax = 0x1;
464 (void) __cpuid_insn(&cpu_regs);
465 if (!(cpu_regs.cp_ecx & CPUID_INTC_ECX_EST)) {
466 return (B_FALSE);
467 }
468
469 return (B_TRUE);
470 }
471
472 boolean_t
turbo_supported(void)473 turbo_supported(void)
474 {
475 struct cpuid_regs cpu_regs;
476
477 /* Required features */
478 if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
479 !is_x86_feature(x86_featureset, X86FSET_MSR)) {
480 return (B_FALSE);
481 }
482
483 /*
484 * turbo mode supported?
485 */
486 cpu_regs.cp_eax = 0x6;
487 (void) __cpuid_insn(&cpu_regs);
488 if (!(cpu_regs.cp_eax & CPUID_TURBO_SUPPORT)) {
489 return (B_FALSE);
490 }
491
492 return (B_TRUE);
493 }
494