1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #include <sys/cpuvar.h>
27 #include <sys/psm.h>
28 #include <sys/archsystm.h>
29 #include <sys/apic.h>
30 #include <sys/sunddi.h>
31 #include <sys/ddi_impldefs.h>
32 #include <sys/mach_intr.h>
33 #include <sys/sysmacros.h>
34 #include <sys/trap.h>
35 #include <sys/x86_archext.h>
36 #include <sys/privregs.h>
37 #include <sys/psm_common.h>
38
39 /* Function prototypes of local apic and X2APIC */
40 static uint64_t local_apic_read(uint32_t reg);
41 static void local_apic_write(uint32_t reg, uint64_t value);
42 static int get_local_apic_pri(void);
43 static void local_apic_write_task_reg(uint64_t value);
44 static void local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
45 static uint64_t local_x2apic_read(uint32_t msr);
46 static void local_x2apic_write(uint32_t msr, uint64_t value);
47 static int get_local_x2apic_pri(void);
48 static void local_x2apic_write_task_reg(uint64_t value);
49 static void local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
50
51 /*
52 * According to the X2APIC specification:
53 *
54 * xAPIC global enable X2APIC enable Description
55 * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10])
56 * -----------------------------------------------------------
57 * 0 0 APIC is disabled
58 * 0 1 Invalid
59 * 1 0 APIC is enabled in xAPIC mode
60 * 1 1 APIC is enabled in X2APIC mode
61 * -----------------------------------------------------------
62 */
63 int x2apic_enable = 1;
64 int apic_mode = LOCAL_APIC; /* Default mode is Local APIC */
65
66 /* Uses MMIO (Memory Mapped IO) */
67 static apic_reg_ops_t local_apic_regs_ops = {
68 local_apic_read,
69 local_apic_write,
70 get_local_apic_pri,
71 local_apic_write_task_reg,
72 local_apic_write_int_cmd,
73 apic_send_EOI,
74 };
75
76 /* X2APIC : Uses RDMSR/WRMSR instructions to access APIC registers */
77 static apic_reg_ops_t x2apic_regs_ops = {
78 local_x2apic_read,
79 local_x2apic_write,
80 get_local_x2apic_pri,
81 local_x2apic_write_task_reg,
82 local_x2apic_write_int_cmd,
83 apic_send_EOI,
84 };
85
86 int apic_have_32bit_cr8 = 0;
87
88 /* The default ops is local APIC (Memory Mapped IO) */
89 apic_reg_ops_t *apic_reg_ops = &local_apic_regs_ops;
90
91 /*
92 * APIC register ops related data sturctures and functions.
93 */
94 void apic_send_EOI();
95 void apic_send_directed_EOI(uint32_t irq);
96
97 #define X2APIC_CPUID_BIT 21
98 #define X2APIC_ENABLE_BIT 10
99
100 /*
101 * Local APIC Implementation
102 */
103 static uint64_t
local_apic_read(uint32_t reg)104 local_apic_read(uint32_t reg)
105 {
106 return ((uint32_t)apicadr[reg]);
107 }
108
109 static void
local_apic_write(uint32_t reg,uint64_t value)110 local_apic_write(uint32_t reg, uint64_t value)
111 {
112 apicadr[reg] = (uint32_t)value;
113 }
114
115 static int
get_local_apic_pri(void)116 get_local_apic_pri(void)
117 {
118 #if defined(__amd64)
119 return ((int)getcr8());
120 #else
121 if (apic_have_32bit_cr8)
122 return ((int)getcr8());
123 return (apicadr[APIC_TASK_REG]);
124 #endif
125 }
126
127 static void
local_apic_write_task_reg(uint64_t value)128 local_apic_write_task_reg(uint64_t value)
129 {
130 #if defined(__amd64)
131 setcr8((ulong_t)(value >> APIC_IPL_SHIFT));
132 #else
133 if (apic_have_32bit_cr8)
134 setcr8((ulong_t)(value >> APIC_IPL_SHIFT));
135 else
136 apicadr[APIC_TASK_REG] = (uint32_t)value;
137 #endif
138 }
139
140 static void
local_apic_write_int_cmd(uint32_t cpu_id,uint32_t cmd1)141 local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
142 {
143 apicadr[APIC_INT_CMD2] = cpu_id << APIC_ICR_ID_BIT_OFFSET;
144 apicadr[APIC_INT_CMD1] = cmd1;
145 }
146
147 /*
148 * X2APIC Implementation.
149 */
150 static uint64_t
local_x2apic_read(uint32_t msr)151 local_x2apic_read(uint32_t msr)
152 {
153 uint64_t i;
154
155 i = (uint64_t)(rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)) & 0xffffffff);
156 return (i);
157 }
158
159 static void
local_x2apic_write(uint32_t msr,uint64_t value)160 local_x2apic_write(uint32_t msr, uint64_t value)
161 {
162 uint64_t tmp;
163
164 if (msr != APIC_EOI_REG) {
165 tmp = rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2));
166 tmp = (tmp & 0xffffffff00000000) | value;
167 } else {
168 tmp = 0;
169 }
170
171 wrmsr((REG_X2APIC_BASE_MSR + (msr >> 2)), tmp);
172 }
173
174 static int
get_local_x2apic_pri(void)175 get_local_x2apic_pri(void)
176 {
177 return (rdmsr(REG_X2APIC_BASE_MSR + (APIC_TASK_REG >> 2)));
178 }
179
180 static void
local_x2apic_write_task_reg(uint64_t value)181 local_x2apic_write_task_reg(uint64_t value)
182 {
183 X2APIC_WRITE(APIC_TASK_REG, value);
184 }
185
186 static void
local_x2apic_write_int_cmd(uint32_t cpu_id,uint32_t cmd1)187 local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
188 {
189 wrmsr((REG_X2APIC_BASE_MSR + (APIC_INT_CMD1 >> 2)),
190 (((uint64_t)cpu_id << 32) | cmd1));
191 }
192
193 /*ARGSUSED*/
194 void
apic_send_EOI(uint32_t irq)195 apic_send_EOI(uint32_t irq)
196 {
197 apic_reg_ops->apic_write(APIC_EOI_REG, 0);
198 }
199
200 /*
201 * Support for Directed EOI capability is available in both the xAPIC
202 * and x2APIC mode.
203 */
204 void
apic_send_directed_EOI(uint32_t irq)205 apic_send_directed_EOI(uint32_t irq)
206 {
207 uchar_t ioapicindex;
208 uchar_t vector;
209 apic_irq_t *apic_irq;
210 short intr_index;
211
212 /*
213 * Following the EOI to the local APIC unit, perform a directed
214 * EOI to the IOxAPIC generating the interrupt by writing to its
215 * EOI register.
216 *
217 * A broadcast EOI is not generated.
218 */
219 apic_reg_ops->apic_write(APIC_EOI_REG, 0);
220
221 apic_irq = apic_irq_table[irq];
222 while (apic_irq) {
223 intr_index = apic_irq->airq_mps_intr_index;
224 if (intr_index == ACPI_INDEX || intr_index >= 0) {
225 ioapicindex = apic_irq->airq_ioapicindex;
226 vector = apic_irq->airq_vector;
227 ioapic_write_eoi(ioapicindex, vector);
228 }
229 apic_irq = apic_irq->airq_next;
230 }
231 }
232
233 int
apic_detect_x2apic(void)234 apic_detect_x2apic(void)
235 {
236 struct cpuid_regs cp;
237
238 if (x2apic_enable == 0)
239 return (0);
240
241 cp.cp_eax = 1;
242 (void) __cpuid_insn(&cp);
243
244 return ((cp.cp_ecx & (0x1 << X2APIC_CPUID_BIT)) ? 1 : 0);
245 }
246
247 void
apic_enable_x2apic(void)248 apic_enable_x2apic(void)
249 {
250 uint64_t apic_base_msr;
251
252 if (apic_local_mode() == LOCAL_X2APIC) {
253 /* BIOS apparently has enabled X2APIC */
254 if (apic_mode != LOCAL_X2APIC)
255 x2apic_update_psm();
256 return;
257 }
258
259 /*
260 * This is the first time we are enabling X2APIC on this CPU
261 */
262 apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
263 apic_base_msr = apic_base_msr | (0x1 << X2APIC_ENABLE_BIT);
264 wrmsr(REG_APIC_BASE_MSR, apic_base_msr);
265
266 if (apic_mode != LOCAL_X2APIC)
267 x2apic_update_psm();
268 }
269
270 /*
271 * Determine which mode the current CPU is in. See the table above.
272 * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10])
273 */
274 int
apic_local_mode(void)275 apic_local_mode(void)
276 {
277 uint64_t apic_base_msr;
278 int bit = ((0x1 << (X2APIC_ENABLE_BIT + 1)) |
279 (0x1 << X2APIC_ENABLE_BIT));
280
281 apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
282
283 if ((apic_base_msr & bit) == bit)
284 return (LOCAL_X2APIC);
285 else
286 return (LOCAL_APIC);
287 }
288
289 void
apic_set_directed_EOI_handler()290 apic_set_directed_EOI_handler()
291 {
292 apic_reg_ops->apic_send_eoi = apic_send_directed_EOI;
293 }
294
295 int
apic_directed_EOI_supported()296 apic_directed_EOI_supported()
297 {
298 uint32_t ver;
299
300 ver = apic_reg_ops->apic_read(APIC_VERS_REG);
301 if (ver & APIC_DIRECTED_EOI_BIT)
302 return (1);
303
304 return (0);
305 }
306
307 /*
308 * Change apic_reg_ops depending upon the apic_mode.
309 */
310 void
apic_change_ops()311 apic_change_ops()
312 {
313 if (apic_mode == LOCAL_APIC)
314 apic_reg_ops = &local_apic_regs_ops;
315 else if (apic_mode == LOCAL_X2APIC)
316 apic_reg_ops = &x2apic_regs_ops;
317 }
318
319 /*
320 * Generates an interprocessor interrupt to another CPU when X2APIC mode is
321 * enabled.
322 */
323 void
x2apic_send_ipi(int cpun,int ipl)324 x2apic_send_ipi(int cpun, int ipl)
325 {
326 int vector;
327 ulong_t flag;
328
329 ASSERT(apic_mode == LOCAL_X2APIC);
330
331 /*
332 * With X2APIC, Intel relaxed the semantics of the
333 * WRMSR instruction such that references to the X2APIC
334 * MSR registers are no longer serializing instructions.
335 * The code that initiates IPIs assumes that some sort
336 * of memory serialization occurs. The old APIC code
337 * did a write to uncachable memory mapped registers.
338 * Any reference to uncached memory is a serializing
339 * operation. To mimic those semantics here, we do an
340 * atomic operation, which translates to a LOCK OR instruction,
341 * which is serializing.
342 */
343 atomic_or_ulong(&flag, 1);
344
345 vector = apic_resv_vector[ipl];
346
347 flag = intr_clear();
348
349 /*
350 * According to X2APIC specification in section '2.3.5.1' of
351 * Interrupt Command Register Semantics, the semantics of
352 * programming Interrupt Command Register to dispatch an interrupt
353 * is simplified. A single MSR write to the 64-bit ICR is required
354 * for dispatching an interrupt. Specifically with the 64-bit MSR
355 * interface to ICR, system software is not required to check the
356 * status of the delivery status bit prior to writing to the ICR
357 * to send an IPI. With the removal of the Delivery Status bit,
358 * system software no longer has a reason to read the ICR. It remains
359 * readable only to aid in debugging.
360 */
361 #ifdef DEBUG
362 APIC_AV_PENDING_SET();
363 #endif /* DEBUG */
364
365 if ((cpun == psm_get_cpu_id())) {
366 X2APIC_WRITE(X2APIC_SELF_IPI, vector);
367 } else {
368 apic_reg_ops->apic_write_int_cmd(
369 apic_cpus[cpun].aci_local_id, vector);
370 }
371
372 intr_restore(flag);
373 }
374
375 /*
376 * Generates IPI to another CPU depending on the local APIC mode.
377 * apic_send_ipi() and x2apic_send_ipi() depends on the configured
378 * mode of the local APIC, but that may not match the actual mode
379 * early in CPU startup.
380 *
381 * Any changes made to this routine must be accompanied by similar
382 * changes to apic_send_ipi().
383 */
384 void
apic_common_send_ipi(int cpun,int ipl)385 apic_common_send_ipi(int cpun, int ipl)
386 {
387 int vector;
388 ulong_t flag;
389 int mode = apic_local_mode();
390
391 if (mode == LOCAL_X2APIC) {
392 x2apic_send_ipi(cpun, ipl);
393 return;
394 }
395
396 ASSERT(mode == LOCAL_APIC);
397
398 vector = apic_resv_vector[ipl];
399 ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
400 flag = intr_clear();
401 while (local_apic_regs_ops.apic_read(APIC_INT_CMD1) & AV_PENDING)
402 apic_ret();
403 local_apic_regs_ops.apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
404 vector);
405 intr_restore(flag);
406 }
407